Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hdfs.server.hightidenode; import java.io.IOException; import java.io.FileNotFoundException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.LinkedList; import java.util.Iterator; import java.util.Arrays; import java.util.Map; import java.util.Random; import java.util.Set; import java.util.HashSet; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.Date; import java.text.SimpleDateFormat; import java.lang.Thread; import java.lang.InterruptedException; import java.net.InetSocketAddress; import java.net.URI; import org.xml.sax.SAXException; import javax.xml.parsers.ParserConfigurationException; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.ipc.*; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Daemon; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.fs.HarFileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.hdfs.protocol.HighTideProtocol; import org.apache.hadoop.hdfs.protocol.PolicyInfo; import org.apache.hadoop.hdfs.protocol.PolicyInfo.PathInfo; import org.apache.hadoop.hdfs.server.hightidenode.metrics.HighTideNodeMetrics; /** * A {@link HighTideNode} that implements */ public class HighTideNode implements HighTideProtocol { static { Configuration.addDefaultResource("hdfs-default.xml"); Configuration.addDefaultResource("hdfs-site.xml"); } public static final Log LOG = LogFactory.getLog("org.apache.hadoop.hightidenode.HighTideNode"); public static final long SLEEP_TIME = 10000L; // 10 seconds public static final int DEFAULT_PORT = 60100; public static final String HIGHTIDE_FULLSYNC_INTERVAL = "hightide.fullsync.interval.seconds"; public static final long HIGHTIDE_FULLSYNC_INTERVAL_DEFAULT = 60 * 60; // 1 hour public static final SimpleDateFormat dateForm = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); /** RPC server */ private Server server; /** RPC server address */ private InetSocketAddress serverAddress = null; /** only used for testing purposes */ private boolean stopRequested = false; /** Configuration Manager */ private ConfigManager configMgr; /** hadoop configuration */ private Configuration conf; protected boolean initialized; // Are we initialized? protected volatile boolean running; // Are we running? /** Deamon thread to find missing blocks */ Daemon triggerThread = null; /** Daemon thread to fix corrupt files */ public FileFixer fileFixer = null; Daemon fileFixerThread = null; static HighTideNodeMetrics myMetrics; // statistics about replicas fixed public static class Statistics { long numProcessedBlocks; // total blocks encountered in namespace long processedSize; // disk space occupied by all blocks public void clear() { numProcessedBlocks = 0; processedSize = 0; } public String toString() { String msg = " numProcessedBlocks = " + numProcessedBlocks + " processedSize = " + processedSize; return msg; } } // Startup options static public enum StartupOption { TEST("-test"), REGULAR("-regular"); private String name = null; private StartupOption(String arg) { this.name = arg; } public String getName() { return name; } } /** * Start HighTideNode. * <p> * The hightidenode-node can be started with one of the following startup options: * <ul> * <li>{@link StartupOption#REGULAR REGULAR} - normal hightidenode node startup</li> * </ul> * The option is passed via configuration field: * <tt>fs.hightidenodenode.startup</tt> * * The conf will be modified to reflect the actual ports on which * the HighTideNode is up and running if the user passes the port as * <code>zero</code> in the conf. * * @param conf confirguration * @throws IOException */ HighTideNode(Configuration conf) throws IOException { try { initialize(conf); } catch (IOException e) { LOG.error(StringUtils.stringifyException(e)); this.stop(); throw e; } catch (Exception e) { LOG.error(StringUtils.stringifyException(e)); this.stop(); throw new IOException(e); } } public long getProtocolVersion(String protocol, long clientVersion) throws IOException { if (protocol.equals(HighTideProtocol.class.getName())) { return HighTideProtocol.versionID; } else { throw new IOException("Unknown protocol to hightide node: " + protocol); } } public ProtocolSignature getProtocolSignature(String protocol, long clientVersion, int clientMethodsHash) throws IOException { return ProtocolSignature.getProtocolSignature(this, protocol, clientVersion, clientMethodsHash); } /** * Wait for service to finish. * (Normally, it runs forever.) */ public void join() { try { if (server != null) server.join(); if (triggerThread != null) triggerThread.join(); if (fileFixerThread != null) fileFixerThread.join(); } catch (InterruptedException ie) { // do nothing } } /** * Stop all HighTideNode threads and wait for all to finish. */ public void stop() { if (stopRequested) { return; } stopRequested = true; running = false; if (server != null) server.stop(); if (triggerThread != null) triggerThread.interrupt(); if (fileFixer != null) fileFixer.shutdown(); if (fileFixerThread != null) fileFixerThread.interrupt(); if (myMetrics != null) { myMetrics.shutdown(); } } private static InetSocketAddress getAddress(String address) { return NetUtils.createSocketAddr(address); } public static InetSocketAddress getAddress(Configuration conf) { String nodeport = conf.get("hightidenode.server.address"); if (nodeport == null) { nodeport = "localhost:" + DEFAULT_PORT; } return getAddress(nodeport); } public InetSocketAddress getListenerAddress() { return server.getListenerAddress(); } private void initialize(Configuration conf) throws IOException, SAXException, InterruptedException, HighTideConfigurationException, ClassNotFoundException, ParserConfigurationException { this.conf = conf; InetSocketAddress socAddr = HighTideNode.getAddress(conf); int handlerCount = conf.getInt("fs.hightidenodenode.handler.count", 10); // read in the configuration configMgr = new ConfigManager(conf); configMgr.reloadConfigsIfNecessary(); configMgr.startReload(); // create Metrics object myMetrics = new HighTideNodeMetrics(conf, this); // create rpc server this.server = RPC.getServer(this, socAddr.getAddress().getHostAddress(), socAddr.getPort(), handlerCount, false, conf); // The rpc-server port can be ephemeral... ensure we have the correct info this.serverAddress = this.server.getListenerAddress(); LOG.info("HighTideNode up at: " + this.serverAddress); initialized = true; running = true; this.server.start(); // start RPC server this.fileFixer = new FileFixer(conf); this.fileFixerThread = new Daemon(this.fileFixer); fileFixer.setPolicyInfo(configMgr.getAllPolicies()); this.fileFixerThread.start(); // start the deamon thread to resync if needed this.triggerThread = new Daemon(new TriggerMonitor()); this.triggerThread.start(); } /** * Sync up on hightidenode restart */ class TriggerMonitor implements Runnable { private Map<String, Long> scanTimes = new HashMap<String, Long>(); private Map<String, DirectoryTraversal> scanState = new HashMap<String, DirectoryTraversal>(); public void run() { while (running) { try { doFullSync(); } catch (IOException e) { LOG.info("Exception in doFullSync. " + StringUtils.stringifyException(e)); } long sleepTime = conf.getLong(HIGHTIDE_FULLSYNC_INTERVAL, HIGHTIDE_FULLSYNC_INTERVAL_DEFAULT) * 1000; try { Thread.sleep(sleepTime); } catch (InterruptedException e) { LOG.info("InterrupedException in TriggerMonitor.run."); } } } } /** * Full sync of all policies */ void doFullSync() throws IOException { for (PolicyInfo pinfo : configMgr.getAllPolicies()) { doFullSync(pinfo); } } /** * Full sync of specified policy */ void doFullSync(PolicyInfo pinfo) throws IOException { Path srcPath = pinfo.getSrcPath(); LOG.info("Starting fullsync of srcPath " + srcPath); FileSystem srcFs = srcPath.getFileSystem(pinfo.getConf()); int srcRepl = Integer.parseInt(pinfo.getProperty("replication")); long modTimePeriod = Long.parseLong(pinfo.getProperty("modTimePeriod")); long now = HighTideNode.now(); // traverse all files inside the subtree rooted at srcPath List<FileStatus> slist = new ArrayList<FileStatus>(); slist.add(srcFs.getFileStatus(srcPath)); DirectoryTraversal traverse = new DirectoryTraversal(srcFs, slist); while (true) { FileStatus sstat = traverse.getNextFile(); if (sstat == null) { break; // done checking all files } // we always follow the order of first changing the replication // on the destination files before we update the repl factor of // the source file. So, it is safe to make the following check // and avoid checking the destination files if the source file // already is at the specified replication factor. if (sstat.getReplication() == srcRepl) { continue; } // if the file has been updated recently, then do not // do anything to it if (sstat.getModificationTime() + modTimePeriod > now) { continue; } // find the suffix in the srcPath that is mapped to the destination srcPath = sstat.getPath(); String[] splits = srcPath.toString().split(pinfo.getSrcPath().toString()); String suffix = splits[1]; // match each pair of src and destination paths boolean match = true; for (PathInfo destPathInfo : pinfo.getDestPaths()) { Path destPath = new Path(destPathInfo.getPath().toString() + suffix); LOG.debug("Comparing " + srcPath + " with " + destPath); int destRepl = Integer.parseInt(destPathInfo.getProperty("replication")); FileSystem destFs = destPath.getFileSystem(pinfo.getConf()); FileStatus dstat = null; try { dstat = destFs.getFileStatus(destPath); } catch (java.io.FileNotFoundException e) { match = false; continue; // ok if the destination does not exist } catch (IOException e) { match = false; LOG.info("Unable to locate matching file in destination " + destPath + StringUtils.stringifyException(e) + ". Ignoring..."); } LOG.info("Matching " + srcPath + " with " + destPath); HighTideNode.getMetrics().filesMatched.inc(); if (dstat.getModificationTime() == sstat.getModificationTime() && dstat.getBlockSize() == sstat.getBlockSize() && dstat.getLen() == sstat.getLen()) { // first reduce the intended replication on the destination path if (dstat.getReplication() > destRepl) { HighTideNode.getMetrics().filesChanged.inc(); long saved = dstat.getLen() * (dstat.getReplication() - destRepl); LOG.info("Changing replication of dest " + destPath + " from " + dstat.getReplication() + " to " + destRepl); destFs.setReplication(dstat.getPath(), (short) destRepl); saved += HighTideNode.getMetrics().savedSize.get(); HighTideNode.getMetrics().savedSize.set(saved); } } else { // one destination path does not match the source match = false; break; } } // if the all the destination paths matched the source, then // reduce repl factor of the source if (match && sstat.getReplication() > srcRepl) { LOG.info("Changing replication of source " + srcPath + " from " + sstat.getReplication() + " to " + srcRepl); HighTideNode.getMetrics().filesChanged.inc(); long saved = sstat.getLen() * (sstat.getReplication() - srcRepl); srcFs.setReplication(sstat.getPath(), (short) srcRepl); saved += HighTideNode.getMetrics().savedSize.get(); HighTideNode.getMetrics().savedSize.set(saved); } } LOG.info("Completed fullsync of srcPath " + srcPath); } /** * Shuts down the HighTideNode */ void shutdown() throws IOException, InterruptedException { configMgr.stopReload(); // stop config reloads fileFixer.shutdown(); // stop block fixer fileFixerThread.interrupt(); server.stop(); // stop http server } /** * Implement HighTideProtocol methods */ /** {@inheritDoc} */ public PolicyInfo[] getAllPolicies() throws IOException { Collection<PolicyInfo> list = configMgr.getAllPolicies(); return list.toArray(new PolicyInfo[list.size()]); } /** * returns my Metrics object */ public static HighTideNodeMetrics getMetrics() { return myMetrics; } /** * Returns current time. */ static long now() { return System.currentTimeMillis(); } private static void printUsage() { System.err.println("Usage: java HighTideNode "); } private static StartupOption parseArguments(String args[]) { int argsLen = (args == null) ? 0 : args.length; StartupOption startOpt = StartupOption.REGULAR; for (int i = 0; i < argsLen; i++) { String cmd = args[i]; // We have to parse command line args in future. } return startOpt; } /** * Convert command line options to configuration parameters */ private static void setStartupOption(Configuration conf, StartupOption opt) { conf.set("fs.hightidenodenode.startup", opt.toString()); } /** * Create an instance of the HighTideNode */ public static HighTideNode createHighTideNode(String argv[], Configuration conf) throws IOException { if (conf == null) { conf = new Configuration(); } StartupOption startOpt = parseArguments(argv); if (startOpt == null) { printUsage(); return null; } setStartupOption(conf, startOpt); HighTideNode node = new HighTideNode(conf); return node; } /** */ public static void main(String argv[]) throws Exception { org.apache.hadoop.hdfs.DnsMonitorSecurityManager.setTheManager(); try { StringUtils.startupShutdownMessage(HighTideNode.class, argv, LOG); HighTideNode hightidenode = createHighTideNode(argv, null); if (hightidenode != null) { hightidenode.join(); } } catch (Throwable e) { LOG.error(StringUtils.stringifyException(e)); System.exit(-1); } } }