Java tutorial
/** * Licensed to Cloudera, Inc. under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. Cloudera, Inc. licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.cloudera.flume.master; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.Writer; import java.net.SocketException; import java.net.UnknownHostException; import java.util.Map; import java.util.Set; import javax.ws.rs.core.Application; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.cli.PosixParser; import org.apache.thrift.transport.TTransportException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.cloudera.flume.agent.FlumeNode; import com.cloudera.flume.conf.FlumeConfiguration; import com.cloudera.flume.master.flows.FlowConfigManager; import com.cloudera.flume.master.logical.LogicalConfigurationManager; import com.cloudera.flume.reporter.ReportEvent; import com.cloudera.flume.reporter.ReportManager; import com.cloudera.flume.reporter.ReportUtil; import com.cloudera.flume.reporter.Reportable; import com.cloudera.flume.reporter.server.AvroReportServer; import com.cloudera.flume.reporter.server.ThriftReportServer; import com.cloudera.flume.util.FlumeVMInfo; import com.cloudera.flume.util.SystemInfo; import com.cloudera.util.CheckJavaVersion; import com.cloudera.util.NetUtils; import com.cloudera.util.StatusHttpServer; import com.sun.jersey.api.core.DefaultResourceConfig; import com.sun.jersey.spi.container.servlet.ServletContainer; /** * This is a first cut at a server for distributing configurations to different * flume client machines. Right now this is a SPOF and not reliable or * persistent, but could eventually hooked into BDB or ZK or something to make * configurations more reliable. */ public class FlumeMaster implements Reportable { protected static final String ZK_CFG_STORE = "zookeeper"; protected static final String MEMORY_CFG_STORE = "memory"; protected final FlumeConfiguration cfg; static final Logger LOG = LoggerFactory.getLogger(FlumeMaster.class); /** report key -- hostname of this master */ static final String REPORTKEY_HOSTNAME = "hostname"; /** report key -- number of nodes reporting to this master */ static final String REPORTKEY_NODES_REPORTING_COUNT = "nodes_reporting_count"; MasterAdminServer configServer; MasterClientServer controlServer; /* * We create instances of both AvroReportServer and ThriftReportServer, and * start the one defined by the flag flume.report.server.rpc.type in the * configuration file. */ ThriftReportServer thriftReportServer = null; AvroReportServer avroReportServer = null; StatusHttpServer http = null; final boolean doHttp; final CommandManager cmdman; final ConfigurationManager specman; final StatusManager statman; final MasterAckManager ackman; final SystemInfo sysInfo = new SystemInfo(this.uniqueMasterName + "."); final FlumeVMInfo vmInfo = new FlumeVMInfo(this.uniqueMasterName + "."); final String uniqueMasterName; Thread reaper; // This is a static instance for commands and for the web interface to get to. static FlumeMaster instance; /** * Warning - do not use this constructor if you think it has been called * anywhere else! This is also not thread safe. * * TODO(henry): Proper singleton implementation * * TODO (jon): make doHttp a FlumeConfiguraiton option */ public FlumeMaster() { this(FlumeConfiguration.get(), true); } /** * Constructs a FlumeMaster using the default FlumeConfiguration, with the web * server off */ public FlumeMaster(FlumeConfiguration cfg) { this(cfg, false); } /** * Warning - do not use this constructor if you think it has been called * anywhere else! This is also not thread safe. * * TODO(henry): Proper singleton implementation */ public FlumeMaster(FlumeConfiguration cfg, boolean doHttp) { this.cfg = cfg; instance = this; this.uniqueMasterName = "flume-master-" + cfg.getMasterServerId(); this.doHttp = doHttp; this.cmdman = new CommandManager(); ConfigStore cfgStore = createConfigStore(FlumeConfiguration.get()); this.statman = new StatusManager(); // configuration manager translate user entered configs if (FlumeConfiguration.get().getMasterIsDistributed()) { LOG.info("Distributed master, disabling all config translations"); ConfigurationManager base = new ConfigManager(cfgStore); this.specman = base; } else { // TODO (jon) translated configurations cause problems in multi-master // situations. For now we disallow translation. LOG.info("Single master, config translations enabled"); ConfigurationManager base = new ConfigManager(cfgStore); ConfigurationManager flowedFailovers = new FlowConfigManager.FailoverFlowConfigManager(base, statman); this.specman = new LogicalConfigurationManager(flowedFailovers, new ConfigManager(), statman); } if (FlumeConfiguration.get().getMasterIsDistributed()) { this.ackman = new GossipedMasterAckManager(FlumeConfiguration.get()); } else { this.ackman = new MasterAckManager(); } } /** * Completely generic and pluggable Flume master constructor. Used for test * cases. Webserver is by default on. */ public FlumeMaster(CommandManager cmd, ConfigurationManager cfgMan, StatusManager stat, MasterAckManager ack, FlumeConfiguration cfg, boolean doHttp) { instance = this; this.doHttp = doHttp; this.cmdman = cmd; this.specman = cfgMan; this.statman = stat; this.ackman = ack; this.cfg = cfg; this.uniqueMasterName = "flume-master-" + cfg.getMasterServerId(); } /** * Completely generic and pluggable Flume master constructor. Used for test * cases. Webserver is by default off. */ public FlumeMaster(CommandManager cmd, ConfigurationManager cfgMan, StatusManager stat, MasterAckManager ack, FlumeConfiguration cfg) { instance = this; this.doHttp = false; this.cmdman = cmd; this.specman = cfgMan; this.statman = stat; this.ackman = ack; this.cfg = cfg; this.uniqueMasterName = "flume-master-" + cfg.getMasterServerId(); } /** * This hook makes it easy for web apps and jsps to get the current FlumeNode * instance. This is used to test the FlumeNode related jsps. */ synchronized public static FlumeMaster getInstance() { if (instance == null) { instance = new FlumeMaster(); } return instance; } /** * Helper function to parse the configuration to decide which kind of config * store to start */ public static ConfigStore createConfigStore(FlumeConfiguration cfg) { ConfigStore cfgStore; if (cfg.getMasterStore().equals(ZK_CFG_STORE)) { cfgStore = new ZooKeeperConfigStore(); } else if (cfg.getMasterStore().equals(MEMORY_CFG_STORE)) { if (cfg.getMasterIsDistributed()) { throw new IllegalStateException("Can't use non-zookeeper store with " + "distributed Master"); } cfgStore = new MemoryBackedConfigStore(); } else { throw new IllegalArgumentException("Unsupported config store: " + cfg.getMasterStore()); } return cfgStore; } /** * Returns a cmd id number that can be used to check status of the command. */ public long submit(Command cmd) { return cmdman.submit(cmd); } ServletContainer jerseyMasterServlet() { Application app = new DefaultResourceConfig(FlumeMasterResource.getResources()); ServletContainer sc = new ServletContainer(app); return sc; } public void serve() throws IOException { if (cfg.getMasterStore().equals(ZK_CFG_STORE)) { try { ZooKeeperService.getAndInit(cfg); } catch (InterruptedException e) { throw new IOException("Unexpected interrupt when starting ZooKeeper", e); } } ReportManager.get().add(vmInfo); ReportManager.get().add(sysInfo); if (doHttp) { String webPath = FlumeNode.getWebPath(cfg); this.http = new StatusHttpServer("flumeconfig", webPath, "0.0.0.0", cfg.getMasterHttpPort(), false); http.addServlet(jerseyMasterServlet(), "/master/*"); http.start(); } controlServer = new MasterClientServer(this, FlumeConfiguration.get()); configServer = new MasterAdminServer(this, FlumeConfiguration.get()); /* * We instantiate both kinds of report servers below, but no resources are * allocated till we call serve() on them. */ avroReportServer = new AvroReportServer(FlumeConfiguration.get().getReportServerPort()); thriftReportServer = new ThriftReportServer(FlumeConfiguration.get().getReportServerPort()); ReportManager.get().add(this); try { controlServer.serve(); configServer.serve(); /* * Start the Avro/Thrift ReportServer based on the flag set in the * configuration file. */ if (cfg.getReportServerRPC() == cfg.RPC_TYPE_AVRO) { avroReportServer.serve(); } else { thriftReportServer.serve(); } } catch (TTransportException e1) { throw new IOException("Error starting control or config server", e1); } cmdman.start(); ackman.start(); specman.start(); // TODO (jon) clean shutdown reaper = new Thread("Lost node reaper") { @Override public void run() { try { while (true) { Thread.sleep(FlumeConfiguration.get().getConfigHeartbeatPeriod()); statman.checkup(); } } catch (InterruptedException e) { LOG.error("Reaper thread unexpectedly interrupted:" + e.getMessage()); LOG.debug("Lost node reaper unexpectedly interrupted", e); } } }; reaper.start(); } /** * Shutdown all the various servers. */ public void shutdown() { try { if (http != null) { try { http.stop(); } catch (Exception e) { LOG.error("Error stopping FlumeMaster", e); } http = null; } cmdman.stop(); ackman.stop(); if (configServer != null) { configServer.stop(); configServer = null; } if (controlServer != null) { controlServer.stop(); controlServer = null; } /* * Close the reportserver which started. */ if (cfg.getReportServerRPC() == cfg.RPC_TYPE_AVRO) { if (avroReportServer != null) { avroReportServer.stop(); avroReportServer = null; } } else { if (thriftReportServer != null) { thriftReportServer.stop(); thriftReportServer = null; } } specman.stop(); reaper.interrupt(); FlumeConfiguration cfg = FlumeConfiguration.get(); if (cfg.getMasterStore().equals(ZK_CFG_STORE)) { ZooKeeperService.get().shutdown(); } } catch (IOException e) { LOG.error("Exception when shutting down master!", e); } catch (Exception e) { LOG.error("Exception when shutting down master!", e); } } /** * Used by internal web app for generating web page with master status * information. */ public String reportHtml() { try { ByteArrayOutputStream baos = new ByteArrayOutputStream(); OutputStreamWriter w = new OutputStreamWriter(baos); reportHtml(w); w.flush(); return baos.toString(); } catch (IOException e) { LOG.error("html report generation failed", e); } return ""; } /** * Generates html 1.0 data that displays the configuration status of the flume * system. */ public void reportHtml(Writer o) throws IOException { statman.getMetrics().toHtml(o); specman.getMetrics().toHtml(o); cmdman.getMetrics().toHtml(o); } /** * Return a list of the names of any nodes that have been seen. Used by the * web interface to populate choice inputs. */ public Set<String> getKnownNodes() { return statman.getNodeStatuses().keySet(); } public ConfigurationManager getSpecMan() { return specman; } public StatusManager getStatMan() { return statman; } public MasterAckManager getAckMan() { return ackman; } public CommandManager getCmdMan() { return cmdman; } @Override public String getName() { return this.uniqueMasterName; } @Override public ReportEvent getMetrics() { ReportEvent rpt = new ReportEvent(getName()); rpt.setStringMetric(REPORTKEY_HOSTNAME, NetUtils.localhost()); rpt.setLongMetric(REPORTKEY_NODES_REPORTING_COUNT, this.getKnownNodes().size()); return rpt; } @Override public Map<String, Reportable> getSubMetrics() { return ReportUtil.noChildren(); } /** * This returns true if the host running this process is in the list of master * servers. The index is set in the FlumeConfiguration. If the host doesn't * match, false is returned. If the hostnames in the master server list fail * to resolve, an exception is thrown. */ public static boolean inferMasterHostID() throws UnknownHostException, SocketException { String masters = FlumeConfiguration.get().getMasterServers(); String[] mtrs = masters.split(","); int idx = NetUtils.findHostIndex(mtrs); if (idx < 0) { String localhost = NetUtils.localhost(); LOG.error("Attempted to start a master '{}' that is not " + "in the master servers list: '{}'", localhost, mtrs); // localhost ips weren't in the list. return false; } FlumeConfiguration.get().setInt(FlumeConfiguration.MASTER_SERVER_ID, idx); LOG.info("Inferred master server index {}", idx); return true; } /** * This is the method that gets run when bin/flume master is executed. */ public static void main(String[] argv) { FlumeNode.logVersion(LOG); FlumeNode.logEnvironment(LOG); // Make sure the Java version is not older than 1.6 if (!CheckJavaVersion.isVersionOk()) { LOG.error("Exiting because of an old Java version or Java version in bad format"); System.exit(-1); } FlumeConfiguration.hardExitLoadConfig(); // if config file is bad hardexit. CommandLine cmd = null; Options options = new Options(); options.addOption("c", true, "Load config from file"); options.addOption("f", false, "Use fresh (empty) flume configs"); options.addOption("i", true, "Server id (an integer from 0 up)"); try { CommandLineParser parser = new PosixParser(); cmd = parser.parse(options, argv); } catch (ParseException e) { HelpFormatter fmt = new HelpFormatter(); fmt.printHelp("FlumeNode", options, true); System.exit(1); } String nodeconfig = FlumeConfiguration.get().getMasterSavefile(); if (cmd != null && cmd.hasOption("c")) { nodeconfig = cmd.getOptionValue("c"); } if (cmd != null && cmd.hasOption("i")) { // if manually overriden by command line, accept it, live with // consequences. String sid = cmd.getOptionValue("i"); LOG.info("Setting serverid from command line to be " + sid); try { int serverid = Integer.parseInt(cmd.getOptionValue("i")); FlumeConfiguration.get().setInt(FlumeConfiguration.MASTER_SERVER_ID, serverid); } catch (NumberFormatException e) { LOG.error("Couldn't parse server id as integer: " + sid); System.exit(1); } } else { // attempt to auto detect master id. try { if (!inferMasterHostID()) { System.exit(1); } } catch (Exception e) { // master needs to be valid to continue; LOG.error("Unable to resolve host '{}' ", e.getMessage()); System.exit(1); } } // This will instantiate and read FlumeConfiguration - so make sure that // this is *after* we set the MASTER_SERVER_ID above. FlumeMaster config = new FlumeMaster(); LOG.info("Starting flume master on: " + NetUtils.localhost()); LOG.info(" Working Directory is: " + new File(".").getAbsolutePath()); try { boolean autoload = FlumeConfiguration.get().getMasterSavefileAutoload(); try { if (autoload && (cmd == null || (cmd != null && !cmd.hasOption("f")))) { // autoload a config? config.getSpecMan().loadConfigFile(nodeconfig); } } catch (IOException e) { LOG.warn("Could not autoload config from " + nodeconfig + " because " + e.getMessage()); } config.serve(); } catch (IOException e) { LOG.error("IO problem: " + e.getMessage()); LOG.debug("IOException", e); } } public FlumeVMInfo getVMInfo() { return vmInfo; } public SystemInfo getSystemInfo() { return sysInfo; } }