List of usage examples for java.net InetSocketAddress getAddress
public final InetAddress getAddress()
From source file:org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager.java
/** * For generating datanode reports/*w w w.j av a2 s . c om*/ */ public List<DatanodeDescriptor> getDatanodeListForReport(final DatanodeReportType type) { final boolean listLiveNodes = type == DatanodeReportType.ALL || type == DatanodeReportType.LIVE; final boolean listDeadNodes = type == DatanodeReportType.ALL || type == DatanodeReportType.DEAD; final boolean listDecommissioningNodes = type == DatanodeReportType.ALL || type == DatanodeReportType.DECOMMISSIONING; ArrayList<DatanodeDescriptor> nodes; final HostFileManager.HostSet foundNodes = new HostFileManager.HostSet(); final HostFileManager.HostSet includedNodes = hostFileManager.getIncludes(); final HostFileManager.HostSet excludedNodes = hostFileManager.getExcludes(); synchronized (datanodeMap) { nodes = new ArrayList<DatanodeDescriptor>(datanodeMap.size()); for (DatanodeDescriptor dn : datanodeMap.values()) { final boolean isDead = isDatanodeDead(dn); final boolean isDecommissioning = dn.isDecommissionInProgress(); if ((listLiveNodes && !isDead) || (listDeadNodes && isDead) || (listDecommissioningNodes && isDecommissioning)) { nodes.add(dn); } foundNodes.add(HostFileManager.resolvedAddressFromDatanodeID(dn)); } } if (listDeadNodes) { for (InetSocketAddress addr : includedNodes) { if (foundNodes.matchedBy(addr) || excludedNodes.match(addr)) { continue; } // The remaining nodes are ones that are referenced by the hosts // files but that we do not know about, ie that we have never // head from. Eg. an entry that is no longer part of the cluster // or a bogus entry was given in the hosts files // // If the host file entry specified the xferPort, we use that. // Otherwise, we guess that it is the default xfer port. // We can't ask the DataNode what it had configured, because it's // dead. DatanodeDescriptor dn = new DatanodeDescriptor(this.storageMap, new DatanodeID(addr.getAddress().getHostAddress(), addr.getHostName(), "", addr.getPort() == 0 ? defaultXferPort : addr.getPort(), defaultInfoPort, defaultInfoSecurePort, defaultIpcPort)); dn.setLastUpdate(0); // Consider this node dead for reporting nodes.add(dn); } } if (LOG.isDebugEnabled()) { LOG.debug("getDatanodeListForReport with " + "includedNodes = " + hostFileManager.getIncludes() + ", excludedNodes = " + hostFileManager.getExcludes() + ", foundNodes = " + foundNodes + ", nodes = " + nodes); } return nodes; }
From source file:com.mellanox.r4h.MiniDFSCluster.java
/** * Restart a datanode, on the same port if requested * /* w w w. ja va 2s . co m*/ * @param dnprop * the datanode to restart * @param keepPort * whether to use the same port * @return true if restarting is successful * @throws IOException */ public synchronized boolean restartDataNode(DataNodeProperties dnprop, boolean keepPort) throws IOException { Configuration conf = dnprop.conf; String[] args = dnprop.dnArgs; SecureResources secureResources = dnprop.secureResources; Configuration newconf = new HdfsConfiguration(conf); // save cloned config if (keepPort) { InetSocketAddress addr = dnprop.datanode.getXferAddress(); conf.set(DFS_DATANODE_ADDRESS_KEY, addr.getAddress().getHostAddress() + ":" + addr.getPort()); conf.set(DFS_DATANODE_IPC_ADDRESS_KEY, addr.getAddress().getHostAddress() + ":" + dnprop.ipcPort); } DataNode newDn = DataNode.createDataNode(args, conf, secureResources); dataNodes.add(new DataNodeProperties(newDn, newconf, args, secureResources, newDn.getIpcPort())); numDataNodes++; return true; }
From source file:edu.umass.cs.reconfiguration.SQLReconfiguratorDB.java
/** * Helper function for getRemoteCheckpoint above that actually fetches the * reads from the socket and writes to a local file. * //from www .j a v a2 s . com * @param rcGroupName * @param sockAddr * @param remoteFilename * @param fileSize * @return */ private String getRemoteCheckpoint(String rcGroupName, InetSocketAddress sockAddr, String remoteFilename, long fileSize) { synchronized (this.fileSystemLock) { String request = remoteFilename + "\n"; Socket sock = null; FileOutputStream fos = null; String localCPFilename = null; try { sock = new Socket(sockAddr.getAddress(), sockAddr.getPort()); sock.getOutputStream().write(request.getBytes(CHARSET)); InputStream inStream = (sock.getInputStream()); if (!this.createCheckpointFile(localCPFilename = this.getCheckpointFile(rcGroupName))) return null; fos = new FileOutputStream(new File(localCPFilename)); byte[] buf = new byte[1024]; int nread = 0; int nTotalRead = 0; // read from sock, write to file while ((nread = inStream.read(buf)) >= 0) { /* Need to ensure that the read won't block forever if the * remote endpoint crashes ungracefully and there is no * exception triggered here. But this method itself is * currently unused. */ nTotalRead += nread; fos.write(buf, 0, nread); } // check exact expected file size if (nTotalRead != fileSize) localCPFilename = null; } catch (IOException e) { e.printStackTrace(); } finally { try { if (fos != null) fos.close(); if (sock != null) sock.close(); } catch (IOException e) { e.printStackTrace(); } } return localCPFilename; } }
From source file:org.apache.hadoop.hdfs.DFSClient.java
private static boolean isLocalAddress(InetSocketAddress targetAddr) { InetAddress addr = targetAddr.getAddress(); if (localIpAddresses.contains(addr.getHostAddress())) { if (LOG.isTraceEnabled()) { LOG.trace("Address " + targetAddr + " is local"); }//from w w w. j a v a 2s .c o m return true; } // Check if the address is any local or loop back boolean local = addr.isAnyLocalAddress() || addr.isLoopbackAddress(); // Check if the address is defined on any interface if (!local) { try { local = NetworkInterface.getByInetAddress(addr) != null; } catch (SocketException e) { local = false; } } if (LOG.isTraceEnabled()) { LOG.trace("Address " + targetAddr + " is local"); } if (local == true) { localIpAddresses.add(addr.getHostAddress()); } return local; }
From source file:org.apache.hadoop.mapred.MapReduceChildJVM.java
public static List<String> getVMCommand(InetSocketAddress taskAttemptListenerAddr, Task task, String javaHome, String workDir, String logDir, String childTmpDir, ID jvmID) { TaskAttemptID attemptID = task.getTaskID(); JobConf conf = task.conf;/*from ww w . j a v a 2s . co m*/ Vector<String> vargs = new Vector<String>(8); vargs.add("exec"); vargs.add(javaHome + "/bin/java"); // Add child (task) java-vm options. // // The following symbols if present in mapred.{map|reduce}.child.java.opts // value are replaced: // + @taskid@ is interpolated with value of TaskID. // Other occurrences of @ will not be altered. // // Example with multiple arguments and substitutions, showing // jvm GC logging, and start of a passwordless JVM JMX agent so can // connect with jconsole and the likes to watch child memory, threads // and get thread dumps. // // <property> // <name>mapred.map.child.java.opts</name> // <value>-Xmx 512M -verbose:gc -Xloggc:/tmp/@taskid@.gc \ // -Dcom.sun.management.jmxremote.authenticate=false \ // -Dcom.sun.management.jmxremote.ssl=false \ // </value> // </property> // // <property> // <name>mapred.reduce.child.java.opts</name> // <value>-Xmx 1024M -verbose:gc -Xloggc:/tmp/@taskid@.gc \ // -Dcom.sun.management.jmxremote.authenticate=false \ // -Dcom.sun.management.jmxremote.ssl=false \ // </value> // </property> // String javaOpts = getChildJavaOpts(conf, task.isMapTask()); javaOpts = javaOpts.replace("@taskid@", attemptID.toString()); String[] javaOptsSplit = javaOpts.split(" "); // Add java.library.path; necessary for loading native libraries. // // 1. We add the 'cwd' of the task to it's java.library.path to help // users distribute native libraries via the DistributedCache. // 2. The user can also specify extra paths to be added to the // java.library.path via mapred.{map|reduce}.child.java.opts. // String libraryPath = workDir; boolean hasUserLDPath = false; for (int i = 0; i < javaOptsSplit.length; i++) { if (javaOptsSplit[i].startsWith("-Djava.library.path=")) { // TODO: Does the above take care of escaped space chars javaOptsSplit[i] += SYSTEM_PATH_SEPARATOR + libraryPath; hasUserLDPath = true; break; } } if (!hasUserLDPath) { vargs.add("-Djava.library.path=" + libraryPath); } for (int i = 0; i < javaOptsSplit.length; i++) { vargs.add(javaOptsSplit[i]); } if (childTmpDir != null) { vargs.add("-Djava.io.tmpdir=" + childTmpDir); } // Setup the log4j prop long logSize = TaskLog.getTaskLogLength(conf); setupLog4jProperties(vargs, logSize, logDir); if (conf.getProfileEnabled()) { if (conf.getProfileTaskRange(task.isMapTask()).isIncluded(task.getPartition())) { File prof = getTaskLogFile(logDir, TaskLog.LogName.PROFILE); vargs.add(String.format(conf.getProfileParams(), prof.toString())); } } // Add main class and its arguments vargs.add(YarnChild.class.getName()); // main of Child // pass TaskAttemptListener's address vargs.add(taskAttemptListenerAddr.getAddress().getHostAddress()); vargs.add(Integer.toString(taskAttemptListenerAddr.getPort())); vargs.add(attemptID.toString()); // pass task identifier // Finally add the jvmID vargs.add(String.valueOf(jvmID.getId())); vargs.add("1>" + getTaskLogFile(logDir, TaskLog.LogName.STDERR)); vargs.add("2>" + getTaskLogFile(logDir, TaskLog.LogName.STDOUT)); // Final commmand StringBuilder mergedCommand = new StringBuilder(); for (CharSequence str : vargs) { mergedCommand.append(str).append(" "); } Vector<String> vargsFinal = new Vector<String>(1); vargsFinal.add(mergedCommand.toString()); return vargsFinal; }
From source file:edu.umass.cs.reconfiguration.Reconfigurator.java
@SuppressWarnings("unused") @Deprecated/* w ww. java2s .co m*/ private AddressMessenger<JSONObject> initClientMessenger() { AbstractPacketDemultiplexer<JSONObject> pd = null; Messenger<InetSocketAddress, JSONObject> cMsgr = null; try { int myPort = (this.consistentNodeConfig.getNodePort(getMyID())); if (ReconfigurationConfig.getClientFacingPort(myPort) != myPort) { log.log(Level.INFO, "{0} creating client messenger at {1}:{2}", new Object[] { this, this.consistentNodeConfig.getBindAddress(getMyID()), ReconfigurationConfig.getClientFacingPort(myPort) }); MessageNIOTransport<InetSocketAddress, JSONObject> niot = null; InetSocketAddress isa = new InetSocketAddress(this.consistentNodeConfig.getBindAddress(getMyID()), ReconfigurationConfig.getClientFacingPort(myPort)); cMsgr = new JSONMessenger<InetSocketAddress>( niot = new MessageNIOTransport<InetSocketAddress, JSONObject>(isa.getAddress(), isa.getPort(), (pd = new ReconfigurationPacketDemultiplexer()), ReconfigurationConfig.getClientSSLMode())); if (!niot.getListeningSocketAddress().equals(isa)) throw new IOException("Unable to listen on specified socket address at " + isa); pd.register(clientRequestTypes, this); } } catch (IOException e) { e.printStackTrace(); log.severe(this + ": " + e.getMessage()); System.exit(1); } return cMsgr != null ? cMsgr : (AddressMessenger<JSONObject>) this.messenger; }
From source file:edu.umass.cs.reconfiguration.Reconfigurator.java
/** * Initiates clear or SSL client messenger based on {@code ssl}. * // w w w . j av a2 s.c om * @param ssl * @return */ @SuppressWarnings("unchecked") private AddressMessenger<JSONObject> initClientMessenger(boolean ssl) { AbstractPacketDemultiplexer<JSONObject> pd = null; Messenger<InetSocketAddress, JSONObject> cMsgr = null; try { int myPort = (this.consistentNodeConfig.getNodePort(getMyID())); if ((ssl ? getClientFacingSSLPort(myPort) : getClientFacingClearPort(myPort)) != myPort) { log.log(Level.INFO, "{0} creating {1} client messenger at {2}:{3}", new Object[] { this, ssl ? "SSL" : "", this.consistentNodeConfig.getBindAddress(getMyID()), "" + (ssl ? getClientFacingSSLPort(myPort) : getClientFacingClearPort(myPort)) }); AddressMessenger<?> existing = (ssl ? this.messenger.getSSLClientMessenger() : this.messenger.getClientMessenger()); if (existing == null || existing == this.messenger) { MessageNIOTransport<InetSocketAddress, JSONObject> niot = null; InetSocketAddress isa = new InetSocketAddress( this.consistentNodeConfig.getBindAddress(getMyID()), ssl ? getClientFacingSSLPort(myPort) : getClientFacingClearPort(myPort)); cMsgr = new JSONMessenger<InetSocketAddress>( niot = new MessageNIOTransport<InetSocketAddress, JSONObject>(isa.getAddress(), isa.getPort(), (pd = new ReconfigurationPacketDemultiplexer()), ssl ? ReconfigurationConfig.getClientSSLMode() : SSL_MODES.CLEAR)); if (!niot.getListeningSocketAddress().equals(isa)) throw new IOException("Unable to listen on specified socket address at " + isa + "; created messenger listening instead on " + niot.getListeningSocketAddress()); } else if (!ssl) { log.log(Level.INFO, "{0} adding self as demultiplexer to existing {1} client messenger", new Object[] { this, ssl ? "SSL" : "" }); if (this.messenger.getClientMessenger() instanceof Messenger) ((Messenger<NodeIDType, ?>) this.messenger.getClientMessenger()) .addPacketDemultiplexer(pd = new ReconfigurationPacketDemultiplexer()); } else { log.log(Level.INFO, "{0} adding self as demultiplexer to existing {1} client messenger", new Object[] { this, ssl ? "SSL" : "" }); if (this.messenger.getSSLClientMessenger() instanceof Messenger) ((Messenger<NodeIDType, ?>) this.messenger.getSSLClientMessenger()) .addPacketDemultiplexer(pd = new ReconfigurationPacketDemultiplexer()); } assert (pd != null); pd.register(clientRequestTypes, this); } } catch (IOException e) { e.printStackTrace(); log.severe(this + " failed to initialize client messenger: " + e.getMessage()); System.exit(1); } if (cMsgr != null) if (ssl && this.messenger.getSSLClientMessenger() == null) this.messenger.setSSLClientMessenger(cMsgr); else if (!ssl && this.messenger.getClientMessenger() == null) this.messenger.setClientMessenger(cMsgr); return cMsgr != null ? cMsgr : (AddressMessenger<JSONObject>) this.messenger; }
From source file:edu.umass.cs.gigapaxos.PaxosManager.java
/** * @param myAddress//w w w.j av a2 s. c o m * @param ssl * @return {@code this} */ private PaxosManager<NodeIDType> initClientMessenger(InetSocketAddress myAddress, boolean ssl, InterfaceNIOTransport<NodeIDType, ?> nioTransport) { Messenger<InetSocketAddress, JSONObject> cMsgr = null; SSLMessenger<NodeIDType, ?> msgr = (nioTransport instanceof Messenger ? (SSLMessenger<NodeIDType, ?>) nioTransport : null); try { int clientPortOffset = ssl ? Config.getGlobalInt(PC.CLIENT_PORT_SSL_OFFSET) : Config.getGlobalInt(PC.CLIENT_PORT_OFFSET); if (clientPortOffset > 0) { InetSocketAddress myAddressOffsetted = new InetSocketAddress(myAddress.getAddress(), myAddress.getPort() + clientPortOffset); log.log(Level.INFO, "{0} creating client messenger at {1}; (offset={2}{3})", new Object[] { this, myAddressOffsetted, clientPortOffset, ssl ? "/SSL" : "" }); MessageNIOTransport<InetSocketAddress, JSONObject> createdNIOTransport = null; cMsgr = new JSONMessenger<InetSocketAddress>( createdNIOTransport = new MessageNIOTransport<InetSocketAddress, JSONObject>( myAddressOffsetted.getAddress(), myAddressOffsetted.getPort(), /* Client facing demultiplexer is single * threaded to keep clients from overwhelming * the system with request load. */ (Config.getGlobalString(PC.JSON_LIBRARY).equals("org.json") ? new JSONDemultiplexer(0, true) : new FastDemultiplexer( Config.getGlobalInt(PC.CLIENT_DEMULTIPLEXER_THREADS), true)), ssl ? SSLDataProcessingWorker.SSL_MODES .valueOf(Config.getGlobalString(PC.CLIENT_SSL_MODE)) : SSL_MODES.CLEAR)); if (Config.getGlobalBoolean(PC.STRICT_ADDRESS_CHECKS) && !createdNIOTransport.getListeningSocketAddress().equals(myAddressOffsetted)) // Note: will throw false positive exception on EC2 throw new IOException("Unable to listen on specified socket address at " + myAddressOffsetted + " != " + createdNIOTransport.getListeningSocketAddress()); assert (msgr != null); if (ssl) msgr.setSSLClientMessenger(cMsgr); else msgr.setClientMessenger(cMsgr); } } catch (IOException e) { e.printStackTrace(); log.severe(e.getMessage()); System.exit(1); } return this; }
From source file:org.apache.hadoop.hdfs.server.namenode.FileSystemProvider.java
/** * ? ? ?? ? ? ? ? ? ./*from www .j a v a 2 s.com*/ * * @param contentsMap { * * bestNode ? ? ? ?? ? * buttonType ? * chunkSizeToView ? ? ? * clusterName ? * currentContentsBlockSize ? ? Block Size * currentPage ? * dfsBlockSize DFS Block Size * dfsBlockStartOffset DFS Block Start Offset * filePath ? * fileSize ? ? * lastDfsBlockSize Last DFS Block Size * startOffset Start Offset * totalPage ? * } * @return contentsMap */ public Map view(Map contentsMap) { try { String filePath = (String) contentsMap.get("filePath"); FileSystem fs = FileSystem.get(Namenode2Agent.configuration); ContentSummary summary = fs.getContentSummary(new Path(filePath)); long fileSize = summary.getLength(); long dfsBlockSize = Long.parseLong(String.valueOf(contentsMap.get("dfsBlockSize"))); long startOffset = Long.parseLong(String.valueOf(contentsMap.get("startOffset"))); long dfsBlockStartOffset = Long.parseLong(String.valueOf(contentsMap.get("dfsBlockStartOffset"))); int currentContentsBlockSize = Integer .parseInt(String.valueOf(contentsMap.get("currentContentsBlockSize"))); int currentPage = (int) contentsMap.get("currentPage"); int totalPage = Integer.parseInt(String.valueOf(contentsMap.get("totalPage"))); String buttonType = (String) contentsMap.get("buttonType"); long chunkSizeToView = contentsMap.containsKey("chunkSizeToView") ? Long.parseLong(String.valueOf(contentsMap.get("chunkSizeToView"))) : DEFAULT_CHUNK_SIZE; long lastDfsBlockSize = 0; if (fileSize > dfsBlockSize) { if (contentsMap.containsKey("lastDfsBlockSize")) { lastDfsBlockSize = Long.parseLong(String.valueOf(contentsMap.get("lastDfsBlockSize"))); } } DFSClient dfsClient = new DFSClient(fs.getUri(), Namenode2Agent.configuration); if (!FileUtils.pathValidator(filePath)) { throw new ServiceException("Invalid path. Please check the path."); } if (chunkSizeToView <= 0) { chunkSizeToView = DEFAULT_CHUNK_SIZE; } long lastPageChunkSizeToView = fileSize % chunkSizeToView; if (currentPage == 0) { if (fileSize > chunkSizeToView) { totalPage = (int) (fileSize / chunkSizeToView); if (lastPageChunkSizeToView > 0) { totalPage++; } } else { totalPage = 1; } if (fileSize > dfsBlockSize) { long lastDfsBlockStartOffset = fileSize; LocatedBlocks locatedBlocks = dfsClient.getNamenode().getBlockLocations(filePath, lastDfsBlockStartOffset, chunkSizeToView); lastDfsBlockSize = locatedBlocks.getLastLocatedBlock().getBlockSize(); contentsMap.put("lastDfsBlockSize", lastDfsBlockSize); } } // ? ? ?(chunkSizeToView) ? ?? ? ? contentsMap.put("totalPage", totalPage); // BlockPool? DFS Block? ? int dfsBlockCount = (int) (fileSize / dfsBlockSize); long dfsBlockResidue = fileSize / dfsBlockSize; if (dfsBlockResidue > 0) { dfsBlockCount++; } int moveToPage; long viewSize = chunkSizeToView; // File contents range to view for DFS Block in BlockPool /** * CurrentPage? ? ? FirstButton? ?? ? 0 . * * Case 1. Next Button * Case 1.1. ? ? ?? * Case 1.2. ? ?? * * Case 2. Last Button * Case 2.1. ? ?? * * Case 3. Previous Button * Case 3.1. ? ? ?? * Case 3.2. ?? ? ? ?? * Case 3.2.1 ?? ? ?? * Case 3.2.2 ?? ? ? ? * * Case 4 Custom Page * Case 4.1. ? ?? * Case 4.2. ? ?? * Case 4.2. ? ?? * * Case 5. Default Page * Case 5.1 ? ? ?? */ switch (buttonType) { case "nextButton": moveToPage = currentPage + 1; if (moveToPage < totalPage) { startOffset += chunkSizeToView; } else if (moveToPage == totalPage) { startOffset = fileSize - lastPageChunkSizeToView; viewSize = lastPageChunkSizeToView; } break; case "lastButton": moveToPage = totalPage; startOffset = fileSize - lastPageChunkSizeToView; viewSize = lastPageChunkSizeToView; break; case "prevButton": moveToPage = currentPage - 1; if (currentPage < totalPage) { startOffset -= chunkSizeToView; } else if (currentPage == totalPage) { if (moveToPage == 1) { startOffset = 0; } else { startOffset -= chunkSizeToView; } } break; case "customPage": moveToPage = currentPage; if (moveToPage == 1) { startOffset = (long) 0; } else if (moveToPage < totalPage) { startOffset = chunkSizeToView * moveToPage; } else if (moveToPage == totalPage) { startOffset = fileSize - lastPageChunkSizeToView; viewSize = lastPageChunkSizeToView; } break; default: moveToPage = 1; startOffset = (long) 0; // ? chunkSizeToView ?? ? ?? ?. if (fileSize < chunkSizeToView) { viewSize = fileSize; } break; } // ??? ? ? contentsMap.put("currentPage", moveToPage); contentsMap.put("startOffset", startOffset); /** * ? ?? ? ? ?? * ??? ?? (fileSize, blockSize, blockCount, genStamp, location...) . * ? ? ?? DFS Client ? ?. * DFS Pool? startOffset Pool? ? ??? DFS ? ? startOffset ? ? ? ? ? ?. */ LocatedBlocks locatedBlocks = dfsClient.getNamenode().getBlockLocations(filePath, startOffset, viewSize); int nextContentsBlockSize = locatedBlocks.locatedBlockCount(); // DFS Block Size ? chunkSizeToView ?? ? ? long dfsBlockViewCount = dfsBlockSize / chunkSizeToView; long dfsBlockViewResidueSize = dfsBlockSize % chunkSizeToView; if (dfsBlockViewResidueSize > 0) { dfsBlockViewCount++; } List<Long> startOffsetPerDfsBlocks = new ArrayList<>(); List<Long> accumulatedStartOffsetPerDfsBlocks = new ArrayList<>(); List<Long> lastStartOffsetPerDfsBlocks = new ArrayList<>(); List<Long> lastChunkSizePerDfsBlocks = new ArrayList<>(); List<Long> pageCheckPoints = new ArrayList<>(); /** * ? ? DFS Block Size ? * ? ? ?? ? ? ?? Block ID ?. * ID ? startOffset ? locatedBlockList ? ? Block ID ?. * ? LocatedBlockSize ? 2. * ? ?(ChunkSizeToView)? ? ?? ? DFS Block? ?? * ? ? ? ?(currentBlockChunkSizeToView) * ? ? ? ? ?(nextBlockChunkSizeToView)? . * ? Block ID ? ?? ? Block ID? ? ? * ? ? startOffset ? ? Merge . * DFS Block Pool? ?? ?? ? startOffset ? DFS Block? startOffset . * * DFS Block Size = 128 MB (134,217,728 B), StartOffset Range Per DFS Block = 0 ~ 134217727, ChunkSizeToView : 10000 * ex. moveToPage == 13421, locatedBlocks size == 2 * First DFS Block's Last StartOffset : 134210000 * Second DFS Block's First(Accumulated) Offset : 0 ~ 2271 * Second DFS Block's Second StartOffset : 2272 * Second DFS Block's Last StartOffset : 134212272 * Third DFS Block's First(Accumulated) Offset : 0 ~ 4543 * Third DFS Block's Second StartOffset : 4544 */ if (fileSize > dfsBlockSize) { long accumulatedStartOffset; long startOffsetForDfsBlock; long startOffsetForSecondDfsBlock = chunkSizeToView - dfsBlockViewResidueSize; long dfsBlockLastChunkSize = chunkSizeToView; for (int i = 0; i < dfsBlockCount; i++) { accumulatedStartOffset = startOffsetForSecondDfsBlock * i; accumulatedStartOffsetPerDfsBlocks.add(i, accumulatedStartOffset); if (dfsBlockLastChunkSize < startOffsetForSecondDfsBlock) { dfsBlockLastChunkSize += chunkSizeToView; } // ? ? ? ? . long lastDfsBlockLastStartOffset = 0; if (i == dfsBlockCount - 1) { long lastDfsBlockViewCount = lastDfsBlockSize / chunkSizeToView; long lastDfsBlockResidue = lastDfsBlockSize % chunkSizeToView; if (lastDfsBlockResidue < dfsBlockLastChunkSize) { lastDfsBlockViewCount--; } lastDfsBlockLastStartOffset = (lastDfsBlockViewCount * chunkSizeToView) + (chunkSizeToView - dfsBlockLastChunkSize); //47841808 dfsBlockLastChunkSize = lastDfsBlockSize - lastDfsBlockLastStartOffset; } else { dfsBlockLastChunkSize -= startOffsetForSecondDfsBlock; } lastChunkSizePerDfsBlocks.add(i, dfsBlockLastChunkSize); long dfsBlockLastStartOffset; if (i == dfsBlockCount - 1) { dfsBlockLastStartOffset = lastDfsBlockLastStartOffset; } else { dfsBlockLastStartOffset = dfsBlockSize - dfsBlockLastChunkSize; } lastStartOffsetPerDfsBlocks.add(i, dfsBlockLastStartOffset); startOffsetForDfsBlock = dfsBlockLastStartOffset % chunkSizeToView; startOffsetPerDfsBlocks.add(i, startOffsetForDfsBlock); } // ? DFS Block? ? contentsMap.put("accumulatedStartOffsetPerDfsBlocks", accumulatedStartOffsetPerDfsBlocks); contentsMap.put("lastStartOffsetPerDfsBlocks", lastStartOffsetPerDfsBlocks); contentsMap.put("lastChunkSizePerDfsBlocks", lastChunkSizePerDfsBlocks); contentsMap.put("startOffsetPerDfsBlocks", startOffsetPerDfsBlocks); long firstPageCheckPoint = dfsBlockSize / chunkSizeToView; long pageCheckPoint = 0; long pageCheckChunkSizeToView = chunkSizeToView; for (int i = 0; i < 15; i++) { pageCheckPoint += firstPageCheckPoint; int j = i; j++; if (j < accumulatedStartOffsetPerDfsBlocks.size()) { if (accumulatedStartOffsetPerDfsBlocks.get(j) > pageCheckChunkSizeToView) { pageCheckChunkSizeToView += chunkSizeToView; pageCheckPoint -= 1; } pageCheckPoints.add(i, pageCheckPoint); pageCheckPoint++; } } // CustomPage ? DFS Block Size ? ? . contentsMap.put("pageCheckPoints", pageCheckPoints); } /** * locatedBlocks ? ? : moveToPage >= dfsBlockViewCount - 1 * * ex. * offsetRange 0 >> moveToPage < dfsBlockViewCount - 1 : 13420 - (13422-1) * offsetRange 1 >> moveToPage == dfsBlockViewCount - 1 : 13421 - (13422-1) * offsetRange 2 >> moveToPage > dfsBlockViewCount - 1 : 13422 - (13422-1) */ int offsetRange = (int) (moveToPage / (dfsBlockViewCount - 1)); LocatedBlock locatedBlock; LocatedBlock nextLocatedBlock = null; long currentBlockLastStartOffset = 0; long currentBlockLastChunkSizeToView = 0; long nextBlockFirstStartOffset = 0; long nextBlockFirstChunkSizeToView = 0; boolean splitViewFlag = false; /** * ?? ? ? ? ? DFS ? * Criteria : DFS Block Size(128MB) and ChunkSizeToView(10000B) * * ? StartOffset ? ? StartOffset(0)? ?? ChunkSizeToView . * currentBlockLastStartOffset ~ nextBlockAccumulatedStartOffset * ex. 134210000 ~ 2272 */ if (nextContentsBlockSize > 1) { splitViewFlag = true; locatedBlock = locatedBlocks.get(0); nextLocatedBlock = locatedBlocks.get(1); dfsBlockStartOffset = startOffsetPerDfsBlocks.get(offsetRange); contentsMap.put("dfsBlockStartOffset", dfsBlockStartOffset); // ? ? startOffset currentBlockLastStartOffset = lastStartOffsetPerDfsBlocks.get(offsetRange - 1); currentBlockLastChunkSizeToView = lastChunkSizePerDfsBlocks.get(offsetRange - 1); nextBlockFirstStartOffset = 0; nextBlockFirstChunkSizeToView = chunkSizeToView - currentBlockLastChunkSizeToView; } else { locatedBlock = locatedBlocks.get(0); } // DFS Block? ? ? ? ? . if (offsetRange < pageCheckPoints.size()) { contentsMap.put("dfsBlockSize", dfsBlockSize); } // ? ? ? ? boolean currentPageSplitViewFlag = false; if (currentContentsBlockSize > 1) { currentPageSplitViewFlag = true; } /** * DFS1 -> DFS0 ?? * currentPageSplitViewFlag true ? dfsBlockStartOffset * ex. 13421 -> 13420 */ if (moveToPage < (dfsBlockViewCount - 1) && (moveToPage + 1) == (dfsBlockViewCount - 1)) { dfsBlockStartOffset = startOffset; } // DFS Block Size ? ? DFS Block ?? ?? StartOffset ? boolean dfsBlockStartOffsetRangeFlag = false; if (fileSize > dfsBlockSize && moveToPage >= dfsBlockViewCount && !splitViewFlag) { dfsBlockStartOffsetRangeFlag = true; } if (dfsBlockStartOffsetRangeFlag) { if (buttonType.equalsIgnoreCase("nextButton")) { if (moveToPage == totalPage) { dfsBlockStartOffset = lastStartOffsetPerDfsBlocks.get(offsetRange); chunkSizeToView = lastChunkSizePerDfsBlocks.get(offsetRange); } else { /** * ? DFS Block startOffset ? ?? ? * ex) DFS Block Size : 128 MB * Second DFS Block StartOffset : 2272 * * ?? ? ? DFS Block? ? startOffset ? . * moveToPage range per DFS block * 0 ~ 13421 : First DFS Block * 13422 ~ 26843 * 26844 ~ 53687 */ if (currentContentsBlockSize < 2) { dfsBlockStartOffset += chunkSizeToView; } } } else if (buttonType.equalsIgnoreCase("prevButton")) { // ?? ? ? ? DFS Block ? ?? ? ? ?? if (currentPageSplitViewFlag) { dfsBlockStartOffset = lastStartOffsetPerDfsBlocks.get(offsetRange - 1); dfsBlockStartOffset -= chunkSizeToView; } else { dfsBlockStartOffset -= chunkSizeToView; } } else if (buttonType.equalsIgnoreCase("customPage")) { // DFS Block Size ? ? splitView . if (moveToPage == totalPage) { dfsBlockStartOffset = lastStartOffsetPerDfsBlocks.get(offsetRange); chunkSizeToView = lastChunkSizePerDfsBlocks.get(offsetRange); } else { long dfsBlockAccumulatedStartOffset = startOffsetPerDfsBlocks.get(offsetRange); long pageCheckPoint = pageCheckPoints.get(offsetRange - 1); long currentPageCount = moveToPage - pageCheckPoint;// 50000-40265=9735 // ?? ? DFS Block ? ? ? ?? if (currentPageCount == 1) { dfsBlockStartOffset = dfsBlockAccumulatedStartOffset; } else { long pageRange = chunkSizeToView; currentPageCount--; if (currentPageCount > 0) { pageRange *= currentPageCount; //97340000, 134210000 } dfsBlockStartOffset = pageRange + dfsBlockAccumulatedStartOffset; // 97346816 } } } else if (buttonType.equalsIgnoreCase("lastButton")) { dfsBlockStartOffset = lastStartOffsetPerDfsBlocks.get(offsetRange); chunkSizeToView = lastChunkSizePerDfsBlocks.get(offsetRange); } contentsMap.put("dfsBlockStartOffset", dfsBlockStartOffset); } contentsMap.put("currentContentsBlockSize", nextContentsBlockSize); contentsMap.put("offsetRange", offsetRange); if (fileSize < dfsBlockSize) { if (moveToPage == totalPage) { chunkSizeToView = lastPageChunkSizeToView; } } /** * Case 1. BestNode , Block ID ? URL . * Case 2. DataNode? BestNode ?. */ InetSocketAddress address; InetSocketAddress nextAddress = null; DatanodeInfo chosenNode; DatanodeInfo nextChosenNode; if (contentsMap.containsKey("bestNode") && !splitViewFlag && !currentPageSplitViewFlag && !dfsBlockStartOffsetRangeFlag && !buttonType.equalsIgnoreCase("customPage")) { String bestNode = (String) contentsMap.get("bestNode"); address = NetUtils.createSocketAddr(bestNode); contentsMap.put("bestNode", bestNode); } else { chosenNode = bestNode(locatedBlock); address = NetUtils.createSocketAddr(chosenNode.getName()); contentsMap.put("bestNode", chosenNode.getName()); if (splitViewFlag) { nextChosenNode = bestNode(nextLocatedBlock); nextAddress = NetUtils.createSocketAddr(nextChosenNode.getName()); contentsMap.put("bestNode", nextChosenNode.getName()); } } /** * DFS File Block Size in HDFS * * ?? DFS ? ?? HDFS? ? * ?? ? ? ? . * ? ? ? locatedBlockCount ? 1 ?. * * ? DFS Block Size * 64 (MB) >> 67,108,864 (B) * 128 (MB) >> 134,217,728 (B) */ String poolId = locatedBlock.getBlock().getBlockPoolId(); long blockId = locatedBlock.getBlock().getBlockId(); long genStamp = locatedBlock.getBlock().getGenerationStamp(); Token<BlockTokenIdentifier> blockToken = locatedBlock.getBlockToken(); DatanodeID datanodeID = new DatanodeID(address.getAddress().getHostAddress(), address.getHostName(), poolId, address.getPort(), 0, 0, 0); Peer peer = dfsClient.newConnectedPeer(address, blockToken, datanodeID); CachingStrategy cachingStrategy = dfsClient.getDefaultReadCachingStrategy(); ExtendedBlock extendedBlock = new ExtendedBlock(poolId, blockId, fileSize, genStamp); String contents; if (splitViewFlag) { String currentBlockContents = streamBlockInAscii(address, blockToken, fileSize, currentBlockLastStartOffset, currentBlockLastChunkSizeToView, fs.getConf(), filePath, dfsClient.getClientName(), extendedBlock, false, peer, datanodeID, cachingStrategy); long nextBlockId = nextLocatedBlock.getBlock().getBlockId(); long nextGenStamp = nextLocatedBlock.getBlock().getGenerationStamp(); Token<BlockTokenIdentifier> nextBlockToken = nextLocatedBlock.getBlockToken(); DatanodeID nextDatanodeID = new DatanodeID(nextAddress.getAddress().getHostAddress(), nextAddress.getHostName(), poolId, nextAddress.getPort(), 0, 0, 0); Peer nextPeer = dfsClient.newConnectedPeer(nextAddress, nextBlockToken, nextDatanodeID); CachingStrategy nextCachingStrategy = dfsClient.getDefaultReadCachingStrategy(); ExtendedBlock nextExtendedBlock = new ExtendedBlock(poolId, nextBlockId, fileSize, nextGenStamp); String nextBlockContents = streamBlockInAscii(nextAddress, nextBlockToken, fileSize, nextBlockFirstStartOffset, nextBlockFirstChunkSizeToView, fs.getConf(), filePath, dfsClient.getClientName(), nextExtendedBlock, false, nextPeer, nextDatanodeID, nextCachingStrategy); // Merge two block's contents contents = currentBlockContents + nextBlockContents; contentsMap.put("startOffset", startOffset); } else { startOffset = dfsBlockStartOffsetRangeFlag || currentPageSplitViewFlag ? dfsBlockStartOffset : startOffset; contents = streamBlockInAscii(address, blockToken, fileSize, startOffset, chunkSizeToView, fs.getConf(), filePath, dfsClient.getClientName(), extendedBlock, false, peer, datanodeID, cachingStrategy); } contentsMap.put("chunkSizeToView", chunkSizeToView); contentsMap.put("lastPageChunkSizeToView", lastPageChunkSizeToView); contentsMap.put("contents", contents); } catch (IOException e) { e.printStackTrace(); } return contentsMap; }
From source file:org.apache.bookkeeper.proto.BookieNettyServer.java
private void listenOn(InetSocketAddress address, BookieSocketAddress bookieAddress) throws InterruptedException { if (!conf.isDisableServerSocketBind()) { ServerBootstrap bootstrap = new ServerBootstrap(); bootstrap.childOption(ChannelOption.ALLOCATOR, new PooledByteBufAllocator(true)); bootstrap.group(eventLoopGroup, eventLoopGroup); bootstrap.childOption(ChannelOption.TCP_NODELAY, conf.getServerTcpNoDelay()); bootstrap.childOption(ChannelOption.SO_LINGER, conf.getServerSockLinger()); bootstrap.childOption(ChannelOption.RCVBUF_ALLOCATOR, new AdaptiveRecvByteBufAllocator(conf.getRecvByteBufAllocatorSizeMin(), conf.getRecvByteBufAllocatorSizeInitial(), conf.getRecvByteBufAllocatorSizeMax())); bootstrap.option(ChannelOption.WRITE_BUFFER_WATER_MARK, new WriteBufferWaterMark( conf.getServerWriteBufferLowWaterMark(), conf.getServerWriteBufferHighWaterMark())); if (eventLoopGroup instanceof EpollEventLoopGroup) { bootstrap.channel(EpollServerSocketChannel.class); } else {/* w ww .j a v a 2s. c o m*/ bootstrap.channel(NioServerSocketChannel.class); } bootstrap.childHandler(new ChannelInitializer<SocketChannel>() { @Override protected void initChannel(SocketChannel ch) throws Exception { synchronized (suspensionLock) { while (suspended) { suspensionLock.wait(); } } BookieSideConnectionPeerContextHandler contextHandler = new BookieSideConnectionPeerContextHandler(); ChannelPipeline pipeline = ch.pipeline(); // For ByteBufList, skip the usual LengthFieldPrepender and have the encoder itself to add it pipeline.addLast("bytebufList", ByteBufList.ENCODER_WITH_SIZE); pipeline.addLast("lengthbaseddecoder", new LengthFieldBasedFrameDecoder(maxFrameSize, 0, 4, 0, 4)); pipeline.addLast("lengthprepender", new LengthFieldPrepender(4)); pipeline.addLast("bookieProtoDecoder", new BookieProtoEncoding.RequestDecoder(registry)); pipeline.addLast("bookieProtoEncoder", new BookieProtoEncoding.ResponseEncoder(registry)); pipeline.addLast("bookieAuthHandler", new AuthHandler.ServerSideHandler( contextHandler.getConnectionPeer(), authProviderFactory)); ChannelInboundHandler requestHandler = isRunning.get() ? new BookieRequestHandler(conf, requestProcessor, allChannels) : new RejectRequestHandler(); pipeline.addLast("bookieRequestHandler", requestHandler); pipeline.addLast("contextHandler", contextHandler); } }); // Bind and start to accept incoming connections Channel listen = bootstrap.bind(address.getAddress(), address.getPort()).sync().channel(); if (listen.localAddress() instanceof InetSocketAddress) { if (conf.getBookiePort() == 0) { conf.setBookiePort(((InetSocketAddress) listen.localAddress()).getPort()); } } } if (conf.isEnableLocalTransport()) { ServerBootstrap jvmBootstrap = new ServerBootstrap(); jvmBootstrap.childOption(ChannelOption.ALLOCATOR, new PooledByteBufAllocator(true)); jvmBootstrap.group(jvmEventLoopGroup, jvmEventLoopGroup); jvmBootstrap.childOption(ChannelOption.TCP_NODELAY, conf.getServerTcpNoDelay()); jvmBootstrap.childOption(ChannelOption.SO_KEEPALIVE, conf.getServerSockKeepalive()); jvmBootstrap.childOption(ChannelOption.SO_LINGER, conf.getServerSockLinger()); jvmBootstrap.childOption(ChannelOption.RCVBUF_ALLOCATOR, new AdaptiveRecvByteBufAllocator(conf.getRecvByteBufAllocatorSizeMin(), conf.getRecvByteBufAllocatorSizeInitial(), conf.getRecvByteBufAllocatorSizeMax())); jvmBootstrap.option(ChannelOption.WRITE_BUFFER_WATER_MARK, new WriteBufferWaterMark( conf.getServerWriteBufferLowWaterMark(), conf.getServerWriteBufferHighWaterMark())); if (jvmEventLoopGroup instanceof DefaultEventLoopGroup) { jvmBootstrap.channel(LocalServerChannel.class); } else if (jvmEventLoopGroup instanceof EpollEventLoopGroup) { jvmBootstrap.channel(EpollServerSocketChannel.class); } else { jvmBootstrap.channel(NioServerSocketChannel.class); } jvmBootstrap.childHandler(new ChannelInitializer<LocalChannel>() { @Override protected void initChannel(LocalChannel ch) throws Exception { synchronized (suspensionLock) { while (suspended) { suspensionLock.wait(); } } BookieSideConnectionPeerContextHandler contextHandler = new BookieSideConnectionPeerContextHandler(); ChannelPipeline pipeline = ch.pipeline(); pipeline.addLast("lengthbaseddecoder", new LengthFieldBasedFrameDecoder(maxFrameSize, 0, 4, 0, 4)); pipeline.addLast("lengthprepender", new LengthFieldPrepender(4)); pipeline.addLast("bookieProtoDecoder", new BookieProtoEncoding.RequestDecoder(registry)); pipeline.addLast("bookieProtoEncoder", new BookieProtoEncoding.ResponseEncoder(registry)); pipeline.addLast("bookieAuthHandler", new AuthHandler.ServerSideHandler( contextHandler.getConnectionPeer(), authProviderFactory)); ChannelInboundHandler requestHandler = isRunning.get() ? new BookieRequestHandler(conf, requestProcessor, allChannels) : new RejectRequestHandler(); pipeline.addLast("bookieRequestHandler", requestHandler); pipeline.addLast("contextHandler", contextHandler); } }); // use the same address 'name', so clients can find local Bookie still discovering them using ZK jvmBootstrap.bind(bookieAddress.getLocalAddress()).sync(); LocalBookiesRegistry.registerLocalBookieAddress(bookieAddress); } }