Java tutorial
/* * Copyright (C) 2012-2016 the Flamingo Community. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hdfs.server.namenode; import org.exem.flamingo.shared.core.exception.ServiceException; import org.exem.flamingo.shared.model.rest.FileInfo; import org.exem.flamingo.shared.util.ExceptionUtils; import org.exem.flamingo.shared.util.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.BlockReader; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.RemoteBlockReader2; import org.apache.hadoop.hdfs.net.Peer; import org.apache.hadoop.hdfs.protocol.*; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.datanode.CachingStrategy; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.token.Token; import org.exem.flamingo.agent.nn.hdfs.HdfsFileInfo; import org.exem.flamingo.agent.nn.hdfs.HdfsFileOnlyInfo; import org.exem.flamingo.shared.util.HdfsUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.helpers.MessageFormatter; import org.springframework.util.FileCopyUtils; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.InetSocketAddress; import java.net.Socket; import java.net.URI; import java.util.*; /** * HDFS ?? HDFS ? Namenode Agnet? HDFS File System Provider. * * @author Myeong Ha, Kim * @since 0.1 */ public class FileSystemProvider { /** * HDFS Default Chunk Size To View */ private static final Long DEFAULT_CHUNK_SIZE = (long) 32768; /** * SLF4J Application Logging */ private Logger logger = LoggerFactory.getLogger(FileSystemProvider.class); /** * SLF4J Exception Logging */ private Logger exceptionLogger = LoggerFactory.getLogger("flamingo.exception"); static Random rand = new Random(); /** * ? ? ? ?? . * * @param path HDFS? ? ? * @param directoryOnly ? ? * @return ? or ? ? */ public List<FileInfo> list(String path, boolean directoryOnly) { try { FileSystem fs = FileSystem.get(Namenode2Agent.configuration); FileStatus[] files; List<FileInfo> fileInfoList = new ArrayList<>(); if (directoryOnly) { files = fs.listStatus(new Path(path)); fileInfoList = new ArrayList<>(files.length); for (FileStatus file : files) { try { if (file.isDirectory()) { fileInfoList.add(new HdfsFileInfo(file, fs.getContentSummary(file.getPath()))); } } catch (Exception ex) { fileInfoList.add(new HdfsFileInfo(file, null)); } } } else { RemoteIterator<LocatedFileStatus> locatedFileStatusIterator = fs.listFiles(new Path(path), false); while (locatedFileStatusIterator.hasNext()) { LocatedFileStatus locatedFileStatus = locatedFileStatusIterator.next(); fileInfoList.add(new HdfsFileOnlyInfo(locatedFileStatus, fs.getContentSummary(locatedFileStatus.getPath()))); } } return fileInfoList; } catch (Exception ex) { throw new ServiceException(" ?? ? .", ex); } } /** * ? ? ? ?? ?. * * @param srcPath HDFS? ? ? * @param username Username * @return true or false */ public boolean mkdir(String srcPath, String username) { if (!FileUtils.pathValidator(srcPath)) { throw new ServiceException("Invalid path. Please check the path."); } if (exists(srcPath)) { throw new ServiceException("? ? ?? ."); } try { FileSystem fs = FileSystem.get(Namenode2Agent.configuration); if (fs.mkdirs(new Path(srcPath))) { setOwnership(new Path(srcPath), username, username); } return true; } catch (IOException ex) { throw new ServiceException(" ? .", ex); } } /** * ? ? ? ?? . * * @param srcPath * @param dstPath ? * @param username Username * @return true or false */ public boolean copy(String srcPath, String dstPath, String username) { if (!FileUtils.pathValidator(srcPath) || !FileUtils.pathValidator(dstPath)) { throw new ServiceException("Invalid path. Please check the path."); } if (!exists(srcPath)) { throw new ServiceException(" ? ?? ."); } if (exists(dstPath)) throw new ServiceException( " ? ?? ?? ? ?? ."); try { FileSystem fs = FileSystem.get(Namenode2Agent.configuration); if (fs.isFile(new Path(srcPath))) { FSDataInputStream fis = fs.open(new Path(srcPath)); FSDataOutputStream fos = fs.create(new Path(dstPath)); FileCopyUtils.copy(fis, fos); IOUtils.closeQuietly(fos); IOUtils.closeQuietly(fis); setOwnership(new Path(srcPath), username, username); } else { FileUtil.copy(fs, new Path(srcPath), fs, new Path(dstPath), false, new Configuration()); } return true; } catch (Exception ex) { throw new ServiceException(" ? ?? .", ex); } } /** * ? ? ? ?? ??. * * @param srcPath * @param dstPath ? * @return true or false */ public boolean move(String srcPath, String dstPath) { if (!FileUtils.pathValidator(srcPath) || !FileUtils.pathValidator(dstPath)) { throw new ServiceException("Invalid path. Please check the path."); } if (!exists(srcPath)) { throw new ServiceException("?? ? ?? ."); } if (!getFileInfo(FileUtils.getPath(dstPath)).isDirectory()) { throw new ServiceException("?? ? ?? ?."); } if (exists(dstPath)) { throw new ServiceException("?? ? ? ?? ."); } try { FileSystem fs = FileSystem.get(Namenode2Agent.configuration); Path from = new Path(srcPath); Path to = new Path(dstPath); return fs.rename(from, to); } catch (Exception ex) { throw new ServiceException(" ? ?? ?? ."); } } /** * ? ? ?? . * * @param srcPath ? ? ?? * @param name ? ? ? * @return true or false */ public boolean rename(String srcPath, String name) { if (!FileUtils.pathValidator(srcPath)) { throw new ServiceException("Invalid path. Please check the path."); } Path oldSrcPath = new Path(srcPath); Path newDstPath = new Path(FileUtils.getPath(srcPath), name); if (exists(newDstPath.toString())) { throw new ServiceException( "? ? ?? ? ?? ."); } try { FileSystem fs = FileSystem.get(Namenode2Agent.configuration); return fs.rename(oldSrcPath, newDstPath); } catch (Exception ex) { throw new ServiceException("? .", ex); } } /** * HDFS? ? ? . * * @param path HDFS? ? ? * @return ?? ? <tt>true</tt>, ? <tt>false</tt> * @throws ServiceException ? ? */ public boolean delete(String path) { if (!FileUtils.pathValidator(path)) { throw new ServiceException("Invalid path. Please check the path."); } if ("/".equals(path)) { throw new ServiceException(" ."); } if (!exists(path)) { throw new ServiceException(" ? ?? ."); } try { FileSystem fs = FileSystem.get(Namenode2Agent.configuration); return fs.delete(new Path(path), true); } catch (Exception ex) { throw new ServiceException(" .", ex); } } /** * HDFS? ?? . * * @param srcPath HDFS? * @param dstPath HDFS? ? ?? ? * @param username ? * @return ?? ? <tt>true</tt>, ? <tt>false</tt> * @throws ServiceException ? ? */ public boolean merge(String srcPath, String dstPath, String username) { if (!FileUtils.pathValidator(srcPath) || !FileUtils.pathValidator(dstPath)) { throw new ServiceException("Invalid path. Please check the path."); } if (!getFileInfo(srcPath).isDirectory()) { throw new ServiceException("? ? ?? ?."); } if (exists(dstPath)) { throw new ServiceException( "? ? ?? ? ?? ."); } try { Path src = new Path(srcPath); Path dst = new Path(dstPath); FileSystem fs = src.getFileSystem(Namenode2Agent.configuration); Path[] srcs = FileUtil.stat2Paths(fs.globStatus(src), src); for (Path source : srcs) { FileUtil.copyMerge(fs, source, fs, dst, false, fs.getConf(), null); } setOwnership(new Path(dstPath), username, username); } catch (IOException e) { e.printStackTrace(); } return true; } /** * HDFS? ? ? . * * @param path HDFS? ? ? * @return ?? ? <tt>FileInfo</tt>, ? <tt>false</tt> * @throws ServiceException ? ? */ public HdfsFileInfo getFileInfo(String path) { try { FileSystem fs = FileSystem.get(Namenode2Agent.configuration); FileStatus fileStatus = fs.getFileStatus(new Path(path)); ContentSummary summary = fs.getContentSummary(new Path(path)); HdfsFileInfo hdfsFileInfo = new HdfsFileInfo(fileStatus, summary); hdfsFileInfo.setDirectory(fileStatus.isDirectory()); hdfsFileInfo.setBlockSize(fileStatus.getBlockSize()); hdfsFileInfo.setReplication(fileStatus.getReplication()); hdfsFileInfo.setDirectoryCount(summary.getDirectoryCount()); hdfsFileInfo.setFileCount(summary.getFileCount()); hdfsFileInfo.setQuota(summary.getQuota()); hdfsFileInfo.setSpaceQuota(summary.getSpaceQuota()); hdfsFileInfo.setSpaceConsumed(summary.getSpaceConsumed()); hdfsFileInfo.setLength(fileStatus.getLen()); return hdfsFileInfo; } catch (Exception ex) { throw new ServiceException(" ? ? .", ex); } } /** * HDFS? ?? . * * @param pathToUpload ?? * @param fullyQualifiedPath ?? * @param content ?? ? * @param username Username * @return */ public boolean save(String pathToUpload, String fullyQualifiedPath, byte[] content, String username) { InputStream is = new ByteArrayInputStream(content); OutputStream os = null; try { FileSystem fs = FileSystem.get(Namenode2Agent.configuration); if (!FileUtils.pathValidator(pathToUpload) || !FileUtils.pathValidator(fullyQualifiedPath)) { throw new ServiceException("Invalid path. Please check the path."); } if (!fs.exists(new Path(pathToUpload))) { throw new ServiceException(" ."); } if (fs.exists(new Path(fullyQualifiedPath))) { throw new ServiceException("?? ? ."); } os = fs.create(new Path(fullyQualifiedPath)); FileCopyUtils.copy(is, os); setOwnership(new Path(fullyQualifiedPath), username, username); return true; } catch (Exception ex) { throw new ServiceException("?? .", ex); } finally { try { if (os != null) os.close(); } catch (Exception ex) { // Ignored } try { is.close(); } catch (Exception ex) { // Ignored } } } public byte[] load(String fullyQualifiedPath) { InputStream content = null; try { FileSystem fs = FileSystem.get(Namenode2Agent.configuration); if (!FileUtils.pathValidator(fullyQualifiedPath)) { throw new ServiceException("Invalid path. Please check the path."); } if (fs.exists(new Path(fullyQualifiedPath))) { throw new ServiceException(" ?? ."); } content = getContent(fullyQualifiedPath); return FileCopyUtils.copyToByteArray(content); } catch (Exception ex) { throw new ServiceException("? ?? .", ex); } finally { IOUtils.closeQuietly(content); } } public InputStream getContent(String path) { if (!exists(path) || !getFileInfo(path).isFile()) { throw new ServiceException("? ?? ?? ."); } try { FileSystem fs = FileSystem.get(Namenode2Agent.configuration); return HdfsUtils.getInputStream(fs, path); } catch (Exception ex) { throw new ServiceException( "? ?? ?? ? . ? ? ? .", ex); } } /** * ? ? ?? ? ? ? ? ? . * * @param contentsMap { * * bestNode ? ? ? ?? ? * buttonType ? * chunkSizeToView ? ? ? * clusterName ? * currentContentsBlockSize ? ? Block Size * currentPage ? * dfsBlockSize DFS Block Size * dfsBlockStartOffset DFS Block Start Offset * filePath ? * fileSize ? ? * lastDfsBlockSize Last DFS Block Size * startOffset Start Offset * totalPage ? * } * @return contentsMap */ public Map view(Map contentsMap) { try { String filePath = (String) contentsMap.get("filePath"); FileSystem fs = FileSystem.get(Namenode2Agent.configuration); ContentSummary summary = fs.getContentSummary(new Path(filePath)); long fileSize = summary.getLength(); long dfsBlockSize = Long.parseLong(String.valueOf(contentsMap.get("dfsBlockSize"))); long startOffset = Long.parseLong(String.valueOf(contentsMap.get("startOffset"))); long dfsBlockStartOffset = Long.parseLong(String.valueOf(contentsMap.get("dfsBlockStartOffset"))); int currentContentsBlockSize = Integer .parseInt(String.valueOf(contentsMap.get("currentContentsBlockSize"))); int currentPage = (int) contentsMap.get("currentPage"); int totalPage = Integer.parseInt(String.valueOf(contentsMap.get("totalPage"))); String buttonType = (String) contentsMap.get("buttonType"); long chunkSizeToView = contentsMap.containsKey("chunkSizeToView") ? Long.parseLong(String.valueOf(contentsMap.get("chunkSizeToView"))) : DEFAULT_CHUNK_SIZE; long lastDfsBlockSize = 0; if (fileSize > dfsBlockSize) { if (contentsMap.containsKey("lastDfsBlockSize")) { lastDfsBlockSize = Long.parseLong(String.valueOf(contentsMap.get("lastDfsBlockSize"))); } } DFSClient dfsClient = new DFSClient(fs.getUri(), Namenode2Agent.configuration); if (!FileUtils.pathValidator(filePath)) { throw new ServiceException("Invalid path. Please check the path."); } if (chunkSizeToView <= 0) { chunkSizeToView = DEFAULT_CHUNK_SIZE; } long lastPageChunkSizeToView = fileSize % chunkSizeToView; if (currentPage == 0) { if (fileSize > chunkSizeToView) { totalPage = (int) (fileSize / chunkSizeToView); if (lastPageChunkSizeToView > 0) { totalPage++; } } else { totalPage = 1; } if (fileSize > dfsBlockSize) { long lastDfsBlockStartOffset = fileSize; LocatedBlocks locatedBlocks = dfsClient.getNamenode().getBlockLocations(filePath, lastDfsBlockStartOffset, chunkSizeToView); lastDfsBlockSize = locatedBlocks.getLastLocatedBlock().getBlockSize(); contentsMap.put("lastDfsBlockSize", lastDfsBlockSize); } } // ? ? ?(chunkSizeToView) ? ?? ? ? contentsMap.put("totalPage", totalPage); // BlockPool? DFS Block? ? int dfsBlockCount = (int) (fileSize / dfsBlockSize); long dfsBlockResidue = fileSize / dfsBlockSize; if (dfsBlockResidue > 0) { dfsBlockCount++; } int moveToPage; long viewSize = chunkSizeToView; // File contents range to view for DFS Block in BlockPool /** * CurrentPage? ? ? FirstButton? ?? ? 0 . * * Case 1. Next Button * Case 1.1. ? ? ?? * Case 1.2. ? ?? * * Case 2. Last Button * Case 2.1. ? ?? * * Case 3. Previous Button * Case 3.1. ? ? ?? * Case 3.2. ?? ? ? ?? * Case 3.2.1 ?? ? ?? * Case 3.2.2 ?? ? ? ? * * Case 4 Custom Page * Case 4.1. ? ?? * Case 4.2. ? ?? * Case 4.2. ? ?? * * Case 5. Default Page * Case 5.1 ? ? ?? */ switch (buttonType) { case "nextButton": moveToPage = currentPage + 1; if (moveToPage < totalPage) { startOffset += chunkSizeToView; } else if (moveToPage == totalPage) { startOffset = fileSize - lastPageChunkSizeToView; viewSize = lastPageChunkSizeToView; } break; case "lastButton": moveToPage = totalPage; startOffset = fileSize - lastPageChunkSizeToView; viewSize = lastPageChunkSizeToView; break; case "prevButton": moveToPage = currentPage - 1; if (currentPage < totalPage) { startOffset -= chunkSizeToView; } else if (currentPage == totalPage) { if (moveToPage == 1) { startOffset = 0; } else { startOffset -= chunkSizeToView; } } break; case "customPage": moveToPage = currentPage; if (moveToPage == 1) { startOffset = (long) 0; } else if (moveToPage < totalPage) { startOffset = chunkSizeToView * moveToPage; } else if (moveToPage == totalPage) { startOffset = fileSize - lastPageChunkSizeToView; viewSize = lastPageChunkSizeToView; } break; default: moveToPage = 1; startOffset = (long) 0; // ? chunkSizeToView ?? ? ?? ?. if (fileSize < chunkSizeToView) { viewSize = fileSize; } break; } // ??? ? ? contentsMap.put("currentPage", moveToPage); contentsMap.put("startOffset", startOffset); /** * ? ?? ? ? ?? * ??? ?? (fileSize, blockSize, blockCount, genStamp, location...) . * ? ? ?? DFS Client ? ?. * DFS Pool? startOffset Pool? ? ??? DFS ? ? startOffset ? ? ? ? ? ?. */ LocatedBlocks locatedBlocks = dfsClient.getNamenode().getBlockLocations(filePath, startOffset, viewSize); int nextContentsBlockSize = locatedBlocks.locatedBlockCount(); // DFS Block Size ? chunkSizeToView ?? ? ? long dfsBlockViewCount = dfsBlockSize / chunkSizeToView; long dfsBlockViewResidueSize = dfsBlockSize % chunkSizeToView; if (dfsBlockViewResidueSize > 0) { dfsBlockViewCount++; } List<Long> startOffsetPerDfsBlocks = new ArrayList<>(); List<Long> accumulatedStartOffsetPerDfsBlocks = new ArrayList<>(); List<Long> lastStartOffsetPerDfsBlocks = new ArrayList<>(); List<Long> lastChunkSizePerDfsBlocks = new ArrayList<>(); List<Long> pageCheckPoints = new ArrayList<>(); /** * ? ? DFS Block Size ? * ? ? ?? ? ? ?? Block ID ?. * ID ? startOffset ? locatedBlockList ? ? Block ID ?. * ? LocatedBlockSize ? 2. * ? ?(ChunkSizeToView)? ? ?? ? DFS Block? ?? * ? ? ? ?(currentBlockChunkSizeToView) * ? ? ? ? ?(nextBlockChunkSizeToView)? . * ? Block ID ? ?? ? Block ID? ? ? * ? ? startOffset ? ? Merge . * DFS Block Pool? ?? ?? ? startOffset ? DFS Block? startOffset . * * DFS Block Size = 128 MB (134,217,728 B), StartOffset Range Per DFS Block = 0 ~ 134217727, ChunkSizeToView : 10000 * ex. moveToPage == 13421, locatedBlocks size == 2 * First DFS Block's Last StartOffset : 134210000 * Second DFS Block's First(Accumulated) Offset : 0 ~ 2271 * Second DFS Block's Second StartOffset : 2272 * Second DFS Block's Last StartOffset : 134212272 * Third DFS Block's First(Accumulated) Offset : 0 ~ 4543 * Third DFS Block's Second StartOffset : 4544 */ if (fileSize > dfsBlockSize) { long accumulatedStartOffset; long startOffsetForDfsBlock; long startOffsetForSecondDfsBlock = chunkSizeToView - dfsBlockViewResidueSize; long dfsBlockLastChunkSize = chunkSizeToView; for (int i = 0; i < dfsBlockCount; i++) { accumulatedStartOffset = startOffsetForSecondDfsBlock * i; accumulatedStartOffsetPerDfsBlocks.add(i, accumulatedStartOffset); if (dfsBlockLastChunkSize < startOffsetForSecondDfsBlock) { dfsBlockLastChunkSize += chunkSizeToView; } // ? ? ? ? . long lastDfsBlockLastStartOffset = 0; if (i == dfsBlockCount - 1) { long lastDfsBlockViewCount = lastDfsBlockSize / chunkSizeToView; long lastDfsBlockResidue = lastDfsBlockSize % chunkSizeToView; if (lastDfsBlockResidue < dfsBlockLastChunkSize) { lastDfsBlockViewCount--; } lastDfsBlockLastStartOffset = (lastDfsBlockViewCount * chunkSizeToView) + (chunkSizeToView - dfsBlockLastChunkSize); //47841808 dfsBlockLastChunkSize = lastDfsBlockSize - lastDfsBlockLastStartOffset; } else { dfsBlockLastChunkSize -= startOffsetForSecondDfsBlock; } lastChunkSizePerDfsBlocks.add(i, dfsBlockLastChunkSize); long dfsBlockLastStartOffset; if (i == dfsBlockCount - 1) { dfsBlockLastStartOffset = lastDfsBlockLastStartOffset; } else { dfsBlockLastStartOffset = dfsBlockSize - dfsBlockLastChunkSize; } lastStartOffsetPerDfsBlocks.add(i, dfsBlockLastStartOffset); startOffsetForDfsBlock = dfsBlockLastStartOffset % chunkSizeToView; startOffsetPerDfsBlocks.add(i, startOffsetForDfsBlock); } // ? DFS Block? ? contentsMap.put("accumulatedStartOffsetPerDfsBlocks", accumulatedStartOffsetPerDfsBlocks); contentsMap.put("lastStartOffsetPerDfsBlocks", lastStartOffsetPerDfsBlocks); contentsMap.put("lastChunkSizePerDfsBlocks", lastChunkSizePerDfsBlocks); contentsMap.put("startOffsetPerDfsBlocks", startOffsetPerDfsBlocks); long firstPageCheckPoint = dfsBlockSize / chunkSizeToView; long pageCheckPoint = 0; long pageCheckChunkSizeToView = chunkSizeToView; for (int i = 0; i < 15; i++) { pageCheckPoint += firstPageCheckPoint; int j = i; j++; if (j < accumulatedStartOffsetPerDfsBlocks.size()) { if (accumulatedStartOffsetPerDfsBlocks.get(j) > pageCheckChunkSizeToView) { pageCheckChunkSizeToView += chunkSizeToView; pageCheckPoint -= 1; } pageCheckPoints.add(i, pageCheckPoint); pageCheckPoint++; } } // CustomPage ? DFS Block Size ? ? . contentsMap.put("pageCheckPoints", pageCheckPoints); } /** * locatedBlocks ? ? : moveToPage >= dfsBlockViewCount - 1 * * ex. * offsetRange 0 >> moveToPage < dfsBlockViewCount - 1 : 13420 - (13422-1) * offsetRange 1 >> moveToPage == dfsBlockViewCount - 1 : 13421 - (13422-1) * offsetRange 2 >> moveToPage > dfsBlockViewCount - 1 : 13422 - (13422-1) */ int offsetRange = (int) (moveToPage / (dfsBlockViewCount - 1)); LocatedBlock locatedBlock; LocatedBlock nextLocatedBlock = null; long currentBlockLastStartOffset = 0; long currentBlockLastChunkSizeToView = 0; long nextBlockFirstStartOffset = 0; long nextBlockFirstChunkSizeToView = 0; boolean splitViewFlag = false; /** * ?? ? ? ? ? DFS ? * Criteria : DFS Block Size(128MB) and ChunkSizeToView(10000B) * * ? StartOffset ? ? StartOffset(0)? ?? ChunkSizeToView . * currentBlockLastStartOffset ~ nextBlockAccumulatedStartOffset * ex. 134210000 ~ 2272 */ if (nextContentsBlockSize > 1) { splitViewFlag = true; locatedBlock = locatedBlocks.get(0); nextLocatedBlock = locatedBlocks.get(1); dfsBlockStartOffset = startOffsetPerDfsBlocks.get(offsetRange); contentsMap.put("dfsBlockStartOffset", dfsBlockStartOffset); // ? ? startOffset currentBlockLastStartOffset = lastStartOffsetPerDfsBlocks.get(offsetRange - 1); currentBlockLastChunkSizeToView = lastChunkSizePerDfsBlocks.get(offsetRange - 1); nextBlockFirstStartOffset = 0; nextBlockFirstChunkSizeToView = chunkSizeToView - currentBlockLastChunkSizeToView; } else { locatedBlock = locatedBlocks.get(0); } // DFS Block? ? ? ? ? . if (offsetRange < pageCheckPoints.size()) { contentsMap.put("dfsBlockSize", dfsBlockSize); } // ? ? ? ? boolean currentPageSplitViewFlag = false; if (currentContentsBlockSize > 1) { currentPageSplitViewFlag = true; } /** * DFS1 -> DFS0 ?? * currentPageSplitViewFlag true ? dfsBlockStartOffset * ex. 13421 -> 13420 */ if (moveToPage < (dfsBlockViewCount - 1) && (moveToPage + 1) == (dfsBlockViewCount - 1)) { dfsBlockStartOffset = startOffset; } // DFS Block Size ? ? DFS Block ?? ?? StartOffset ? boolean dfsBlockStartOffsetRangeFlag = false; if (fileSize > dfsBlockSize && moveToPage >= dfsBlockViewCount && !splitViewFlag) { dfsBlockStartOffsetRangeFlag = true; } if (dfsBlockStartOffsetRangeFlag) { if (buttonType.equalsIgnoreCase("nextButton")) { if (moveToPage == totalPage) { dfsBlockStartOffset = lastStartOffsetPerDfsBlocks.get(offsetRange); chunkSizeToView = lastChunkSizePerDfsBlocks.get(offsetRange); } else { /** * ? DFS Block startOffset ? ?? ? * ex) DFS Block Size : 128 MB * Second DFS Block StartOffset : 2272 * * ?? ? ? DFS Block? ? startOffset ? . * moveToPage range per DFS block * 0 ~ 13421 : First DFS Block * 13422 ~ 26843 * 26844 ~ 53687 */ if (currentContentsBlockSize < 2) { dfsBlockStartOffset += chunkSizeToView; } } } else if (buttonType.equalsIgnoreCase("prevButton")) { // ?? ? ? ? DFS Block ? ?? ? ? ?? if (currentPageSplitViewFlag) { dfsBlockStartOffset = lastStartOffsetPerDfsBlocks.get(offsetRange - 1); dfsBlockStartOffset -= chunkSizeToView; } else { dfsBlockStartOffset -= chunkSizeToView; } } else if (buttonType.equalsIgnoreCase("customPage")) { // DFS Block Size ? ? splitView . if (moveToPage == totalPage) { dfsBlockStartOffset = lastStartOffsetPerDfsBlocks.get(offsetRange); chunkSizeToView = lastChunkSizePerDfsBlocks.get(offsetRange); } else { long dfsBlockAccumulatedStartOffset = startOffsetPerDfsBlocks.get(offsetRange); long pageCheckPoint = pageCheckPoints.get(offsetRange - 1); long currentPageCount = moveToPage - pageCheckPoint;// 50000-40265=9735 // ?? ? DFS Block ? ? ? ?? if (currentPageCount == 1) { dfsBlockStartOffset = dfsBlockAccumulatedStartOffset; } else { long pageRange = chunkSizeToView; currentPageCount--; if (currentPageCount > 0) { pageRange *= currentPageCount; //97340000, 134210000 } dfsBlockStartOffset = pageRange + dfsBlockAccumulatedStartOffset; // 97346816 } } } else if (buttonType.equalsIgnoreCase("lastButton")) { dfsBlockStartOffset = lastStartOffsetPerDfsBlocks.get(offsetRange); chunkSizeToView = lastChunkSizePerDfsBlocks.get(offsetRange); } contentsMap.put("dfsBlockStartOffset", dfsBlockStartOffset); } contentsMap.put("currentContentsBlockSize", nextContentsBlockSize); contentsMap.put("offsetRange", offsetRange); if (fileSize < dfsBlockSize) { if (moveToPage == totalPage) { chunkSizeToView = lastPageChunkSizeToView; } } /** * Case 1. BestNode , Block ID ? URL . * Case 2. DataNode? BestNode ?. */ InetSocketAddress address; InetSocketAddress nextAddress = null; DatanodeInfo chosenNode; DatanodeInfo nextChosenNode; if (contentsMap.containsKey("bestNode") && !splitViewFlag && !currentPageSplitViewFlag && !dfsBlockStartOffsetRangeFlag && !buttonType.equalsIgnoreCase("customPage")) { String bestNode = (String) contentsMap.get("bestNode"); address = NetUtils.createSocketAddr(bestNode); contentsMap.put("bestNode", bestNode); } else { chosenNode = bestNode(locatedBlock); address = NetUtils.createSocketAddr(chosenNode.getName()); contentsMap.put("bestNode", chosenNode.getName()); if (splitViewFlag) { nextChosenNode = bestNode(nextLocatedBlock); nextAddress = NetUtils.createSocketAddr(nextChosenNode.getName()); contentsMap.put("bestNode", nextChosenNode.getName()); } } /** * DFS File Block Size in HDFS * * ?? DFS ? ?? HDFS? ? * ?? ? ? ? . * ? ? ? locatedBlockCount ? 1 ?. * * ? DFS Block Size * 64 (MB) >> 67,108,864 (B) * 128 (MB) >> 134,217,728 (B) */ String poolId = locatedBlock.getBlock().getBlockPoolId(); long blockId = locatedBlock.getBlock().getBlockId(); long genStamp = locatedBlock.getBlock().getGenerationStamp(); Token<BlockTokenIdentifier> blockToken = locatedBlock.getBlockToken(); DatanodeID datanodeID = new DatanodeID(address.getAddress().getHostAddress(), address.getHostName(), poolId, address.getPort(), 0, 0, 0); Peer peer = dfsClient.newConnectedPeer(address, blockToken, datanodeID); CachingStrategy cachingStrategy = dfsClient.getDefaultReadCachingStrategy(); ExtendedBlock extendedBlock = new ExtendedBlock(poolId, blockId, fileSize, genStamp); String contents; if (splitViewFlag) { String currentBlockContents = streamBlockInAscii(address, blockToken, fileSize, currentBlockLastStartOffset, currentBlockLastChunkSizeToView, fs.getConf(), filePath, dfsClient.getClientName(), extendedBlock, false, peer, datanodeID, cachingStrategy); long nextBlockId = nextLocatedBlock.getBlock().getBlockId(); long nextGenStamp = nextLocatedBlock.getBlock().getGenerationStamp(); Token<BlockTokenIdentifier> nextBlockToken = nextLocatedBlock.getBlockToken(); DatanodeID nextDatanodeID = new DatanodeID(nextAddress.getAddress().getHostAddress(), nextAddress.getHostName(), poolId, nextAddress.getPort(), 0, 0, 0); Peer nextPeer = dfsClient.newConnectedPeer(nextAddress, nextBlockToken, nextDatanodeID); CachingStrategy nextCachingStrategy = dfsClient.getDefaultReadCachingStrategy(); ExtendedBlock nextExtendedBlock = new ExtendedBlock(poolId, nextBlockId, fileSize, nextGenStamp); String nextBlockContents = streamBlockInAscii(nextAddress, nextBlockToken, fileSize, nextBlockFirstStartOffset, nextBlockFirstChunkSizeToView, fs.getConf(), filePath, dfsClient.getClientName(), nextExtendedBlock, false, nextPeer, nextDatanodeID, nextCachingStrategy); // Merge two block's contents contents = currentBlockContents + nextBlockContents; contentsMap.put("startOffset", startOffset); } else { startOffset = dfsBlockStartOffsetRangeFlag || currentPageSplitViewFlag ? dfsBlockStartOffset : startOffset; contents = streamBlockInAscii(address, blockToken, fileSize, startOffset, chunkSizeToView, fs.getConf(), filePath, dfsClient.getClientName(), extendedBlock, false, peer, datanodeID, cachingStrategy); } contentsMap.put("chunkSizeToView", chunkSizeToView); contentsMap.put("lastPageChunkSizeToView", lastPageChunkSizeToView); contentsMap.put("contents", contents); } catch (IOException e) { e.printStackTrace(); } return contentsMap; } /** * ?? ? ?? ? ? ?? ?. * * @param blk LocatedBlock * @return ?? */ public static DatanodeInfo bestNode(LocatedBlock blk) { TreeSet<DatanodeInfo> deadNodes = new TreeSet<>(); DatanodeInfo chosenNode = null; int failures = 0; Socket socket = null; DatanodeInfo[] nodes = blk.getLocations(); if (nodes == null || nodes.length == 0) { throw new ServiceException("No nodes contain this block"); } while (socket == null) { if (chosenNode == null) { do { chosenNode = nodes[rand.nextInt(nodes.length)]; } while (deadNodes.contains(chosenNode)); } int index = rand.nextInt(nodes.length); chosenNode = nodes[index]; //just ping to check whether the node is alive InetSocketAddress address = NetUtils .createSocketAddr(chosenNode.getIpAddr() + ":" + chosenNode.getInfoPort()); try { socket = new Socket(); socket.connect(address, HdfsServerConstants.READ_TIMEOUT); socket.setSoTimeout(HdfsServerConstants.READ_TIMEOUT); } catch (IOException e) { deadNodes.add(chosenNode); try { socket.close(); } catch (IOException ex) { ex.printStackTrace(); } socket = null; failures++; } if (failures == nodes.length) { throw new ServiceException("Could not reach the block containing the data. Please try again"); } } try { socket.close(); } catch (IOException e) { e.printStackTrace(); } return chosenNode; } /** * ?? ? ? ? ??. * * @param inetSocketAddress InetSocketAddress * @param blockToken Token<BlockTokenIdentifier> * @param fileSize FileSize * @param startOffset StartOffset * @param chunkSizeToView ChunkSizeToView * @param conf Configuration * @param fileLocation FileLocation * @param clientName ClientName * @param block Block * @param verifyChecksum VerifyChecksum * @param peer Peer * @param datanodeID DataNodeID * @param cachingStrategy CachingStrategy * @return String * @throws IOException */ public String streamBlockInAscii(InetSocketAddress inetSocketAddress, Token<BlockTokenIdentifier> blockToken, long fileSize, long startOffset, long chunkSizeToView, Configuration conf, String fileLocation, String clientName, ExtendedBlock block, boolean verifyChecksum, Peer peer, DatanodeID datanodeID, CachingStrategy cachingStrategy) throws IOException { if (chunkSizeToView == 0) { throw new ServiceException("Cannot read chunk size to view."); } Socket socket = NetUtils.getDefaultSocketFactory(conf).createSocket(); socket.connect(inetSocketAddress, HdfsServerConstants.READ_TIMEOUT); socket.setSoTimeout(HdfsServerConstants.READ_TIMEOUT); BlockReader blockReader = RemoteBlockReader2.newBlockReader(fileLocation, block, blockToken, startOffset, chunkSizeToView, verifyChecksum, clientName, peer, datanodeID, null, cachingStrategy); int amtToRead = (int) Math.min(chunkSizeToView, fileSize); final byte[] buf = new byte[amtToRead]; int readOffset = 0; int retires = 2; while (amtToRead > 0) { int numRead = amtToRead; try { blockReader.readFully(buf, readOffset, amtToRead); } catch (IOException e) { retires--; if (retires == 0) { throw new ServiceException("Could not read data from datanode."); } continue; } amtToRead -= numRead; readOffset += numRead; } blockReader.close(); socket.close(); return new String(buf); } /** * ? ? ? , ? . * * @param permissionMap * @return true or false */ public boolean setPermission(Map permissionMap) { String srcPath = (String) permissionMap.get("currentPath"); String owner = (String) permissionMap.get("owner"); String group = (String) permissionMap.get("group"); String fileStatus = (String) permissionMap.get("fileStatus"); boolean recursiveOwner = (int) permissionMap.get("recursiveOwner") == 1; boolean recursivePermission = (int) permissionMap.get("recursivePermission") == 1; String permission = (String) permissionMap.get("permission"); try { if (srcPath.equalsIgnoreCase("/")) { throw new ServiceException(" ? ."); } if (fileStatus.equalsIgnoreCase("DIRECTORY")) { runChown(recursiveOwner, owner, group, srcPath); runChmod(recursivePermission, permission, srcPath); } else { FileSystem fs = FileSystem.get(Namenode2Agent.configuration); FsPermission fsPermission = new FsPermission(permission); String file = (String) permissionMap.get("file"); if (file.equalsIgnoreCase("/")) { throw new ServiceException(" ? ."); } fs.setOwner(new Path(file), owner, group); fs.setPermission(new Path(file), fsPermission); } } catch (Exception ex) { ex.printStackTrace(); throw new ServiceException("? .", ex); } return true; } /** * HDFS ? ? ?. * * @param hdfsUserMap HDFS ? ? ? ? * @return true or false */ public boolean createUserHome(Map hdfsUserMap) { String hdfsUserHome = (String) hdfsUserMap.get("hdfsUserHome"); String username = (String) hdfsUserMap.get("username"); Path path = new Path(hdfsUserHome); if (!FileUtils.pathValidator(hdfsUserHome)) { throw new ServiceException("Invalid path. Please check the path."); } try { FileSystem fs = FileSystem.get(Namenode2Agent.configuration); if (!fs.mkdirs(path)) { throw new ServiceException("? ? ."); } fs.setOwner(path, username, username); fs.setPermission(path, new FsPermission((short) 00700)); } catch (Exception ex) { throw new ServiceException("? ? ."); } return true; } /** * HDFS ? ? . * * @param hdfsUserHomePath HDFS ? * @return true or false */ public boolean deleteUserHome(String hdfsUserHomePath) { boolean deleted; if (!FileUtils.pathValidator(hdfsUserHomePath)) { throw new ServiceException("Invalid path. Please check the path."); } if (!exists(hdfsUserHomePath)) { throw new ServiceException("? ."); } try { FileSystem fs = FileSystem.get(Namenode2Agent.configuration); Path path = new Path(hdfsUserHomePath); logger.debug("Delete HDFS User Home Path : {}", path); deleted = fs.delete(path, true); if (!deleted) { exceptionLogger.warn("Failed delete HDFS User Home : {}", path); throw new ServiceException("? ."); } return true; } catch (Exception ex) { throw new ServiceException("? ."); } } /** * ? HDFS ? ()? . * * @param recursive ? * @param owner ? * @param group * @param srcPath ? HDFS * @return true or false */ private boolean runChown(boolean recursive, String owner, String group, String srcPath) { try { FsShell fsShell = new FsShell(Namenode2Agent.configuration); String ownership = MessageFormatter.arrayFormat("{}:{}", new String[] { owner, group }).getMessage(); int result; if (recursive) { result = fsShell.run(new String[] { "-chown", "-R", ownership, srcPath }); } else { result = fsShell.run(new String[] { "-chown", ownership, srcPath }); } logger.debug("? HDFS '{}'? ? ?[Recursive : {}].", srcPath, recursive); return result == 0; } catch (Exception ex) { logger.warn( " '{}' HDFS ?? ? .", srcPath); exceptionLogger.warn("{} : {}\n{}", new String[] { ex.getClass().getName(), ex.getMessage(), ExceptionUtils.getFullStackTrace(ex) }); return false; } } /** * ? HDFS ? ()? . * * @param recursive ? * @param permission ? (ex. 777) * @param srcPath ? HDFS * @return true of false */ private boolean runChmod(boolean recursive, String permission, String srcPath) { try { FsShell fsShell = new FsShell(Namenode2Agent.configuration); int result; if (recursive) { result = fsShell.run(new String[] { "-chmod", "-R", permission, srcPath }); } else { result = fsShell.run(new String[] { "-chmod", permission, srcPath }); } logger.debug( "? HDFS '{}'? ? ? ?[Recursive : {}].", srcPath, recursive); return result == 0; } catch (Exception ex) { logger.warn( " '{}' HDFS ?? ? .", srcPath); exceptionLogger.warn("{} : {}\n{}", new String[] { ex.getClass().getName(), ex.getMessage(), ExceptionUtils.getFullStackTrace(ex) }); return false; } } /** * ? ? ?? ?. * * @param path HDFS? ? ? * @return true or false */ public boolean exists(String path) { try { FileSystem fs = FileSystem.get(Namenode2Agent.configuration); return fs.exists(new Path(path)); } catch (Exception ex) { throw new ServiceException(" ? .", ex); } } /** * HDFS ? ? ? ? ?? ? . * * @param srcFullyQualifiedPath ? ?? HDFS * @param linuxUserHome ? ? */ public boolean copyToLocal(String srcFullyQualifiedPath, String dstFullyQualifiedPath, String linuxUserHome, String username) { try { FileSystem fs = FileSystem.get(Namenode2Agent.configuration); Path path = new Path(srcFullyQualifiedPath); Path localPath = new Path(dstFullyQualifiedPath); String mode = "666"; //rw-rw-rw- Configuration conf = new Configuration(); LocalFileSystem lfs = (LocalFileSystem) LocalFileSystem.get(URI.create("file:///"), conf); if (lfs.createNewFile(localPath)) { fs.copyToLocalFile(false, path, new Path(linuxUserHome), false); lfs.setPermission(localPath, new FsPermission(mode)); } else { return false; } return true; } catch (IOException ex) { throw new ServiceException(" ? .", ex); } } /** * ? ? ? ?? ? ?? . * * @param path HDFS? ? ? * @param username Username * @param userGroup UserGroup */ private void setOwnership(Path path, String username, String userGroup) { try { FileSystem fs = FileSystem.get(Namenode2Agent.configuration); fs.setOwner(path, username, userGroup); } catch (IOException ex) { throw new ServiceException("? .", ex); } } }