Java tutorial
/** * Licensed to Cloudera, Inc. under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. Cloudera, Inc. licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.cloudera.flume.master; import java.io.IOException; import java.util.Collection; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.Map.Entry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.cloudera.flume.conf.FlumeConfigData; import com.cloudera.flume.conf.FlumeSpecException; import com.cloudera.flume.reporter.ReportEvent; import com.cloudera.flume.reporter.ReportUtil; import com.cloudera.flume.reporter.Reportable; import com.google.common.base.Preconditions; import com.google.common.collect.Multimap; /** * This translating configuration manager encapsulates the logic for having * multiple configuration translations. There are parent configuration and self * configuration manager. The parent may be a configuration manager that does * storage or another translating manager and is used to hold pre-translated * configurations. The self manager must be a storage configuration manager for * keeping translated configurations. * * If in-place changes are ok, use the same configuration manager as the parent * and self. If not, use a different configuration manager for parent and self. * * Read method calls on a TranslatingConfiguraitonManager always read from the * self manager. Write method calls write to the parent manager and write the * translated versions to the self manager. */ abstract public class TranslatingConfigurationManager implements ConfigurationManager, Translator { static final Logger LOG = LoggerFactory.getLogger(TranslatingConfigurationManager.class); ConfigurationManager parentMan; ConfigurationManager selfMan; /** * Create a new translator where the parent and the self are the same. */ public TranslatingConfigurationManager(ConfigurationManager single) { this.parentMan = single; this.selfMan = single; } /** * Create a new translator with a parent and a self. */ public TranslatingConfigurationManager(ConfigurationManager parent, ConfigurationManager self) { this.parentMan = parent; this.selfMan = self; } /** * {@inheritDoc} * * Gets the translated configuration data. */ @Override synchronized public FlumeConfigData getConfig(String host) { return selfMan.getConfig(host); } /** * {@inheritDoc} setConfig actually writes two entries -- the version which is * the user entered to the parent, and a translated version that a * heartbeating node would read into the self table. */ @Override synchronized public void setConfig(String logicalnode, String flowid, String source, String sink) throws IOException, FlumeSpecException { // if parent and self are different, save original to parent. if (parentMan != selfMan) { parentMan.setConfig(logicalnode, flowid, source, sink); FlumeConfigData fcd = parentMan.getConfig(logicalnode); // parent may have translated source = fcd.getSourceConfig(); sink = fcd.getSinkConfig(); } String xsink = translateSink(logicalnode, sink); String xsource = translateSource(logicalnode, source); // save the translated sink that is sent to the master. selfMan.setConfig(logicalnode, flowid, xsource, xsink); updateAll(); } @Override abstract public String getName(); /** * Internal formatting method for FlumeConfigData and translated * FlumeConfigData */ static void appendHtmlTranslatedFlumeConfigData(StringBuilder html, String name, FlumeConfigData fcd, FlumeConfigData xfcd) { html.append("\n<tr>"); html.append("<td>" + name + "</td>"); FlumeConfigData cfg = fcd; html.append("<td>" + new Date(cfg.timestamp) + "</td>"); html.append("<td>" + cfg.flowID + "</td>"); html.append("<td>" + cfg.sourceConfig + "</td>"); html.append("<td>" + cfg.sinkConfig + "</td>"); if (xfcd != null) { html.append("<td>" + new Date(xfcd.timestamp) + "</td>"); html.append("<td>" + xfcd.sourceConfig + "</td>"); html.append("<td>" + xfcd.sinkConfig + "</td>"); } else { html.append("<td></td><td></td><td></td>"); } html.append("</tr>\n"); } /** * Builds a two html tables that display parent/self configs and logical node * mapping. * * TODO convert to a report, do not depend on this output. */ @Override synchronized public ReportEvent getMetrics() { StringBuilder html = new StringBuilder(); html.append("<h2>Node configuration</h2>\n<table border=\"1\"><tr>" + "<th>Node</th><th>Version</th><th>Flow ID</th><th>Source</th><th>Sink</th>" + "<th>Translated Version</th><th>Translated Source</th><th>Translated Sink</th>" + "</tr>"); Map<String, FlumeConfigData> cfgs = new TreeMap<String, FlumeConfigData>(parentMan.getAllConfigs()); Map<String, FlumeConfigData> xcfgs = new TreeMap<String, FlumeConfigData>(selfMan.getTranslatedConfigs()); for (Entry<String, FlumeConfigData> e : cfgs.entrySet()) { String ln = e.getKey(); appendHtmlTranslatedFlumeConfigData(html, e.getKey(), e.getValue(), xcfgs.get(ln)); } html.append("</table>\n\n"); // a table that has a mapping from physical nodes to logical nodes. html.append("<h2>Physical/Logical Node mapping</h2>\n<table border=\"1\">" + "<tr><th>physical node</th><th>logical node</th></tr>"); Multimap<String, String> nodes = parentMan.getLogicalNodeMap(); synchronized (nodes) { for (Entry<String, Collection<String>> e : nodes.asMap().entrySet()) { ConfigManager.appendHtmlPhysicalLogicalMapping(html, e.getKey(), e.getValue()); } } html.append("</table>\n\n"); return ReportEvent.createLegacyHtmlReport("configs", html.toString()); } // TODO make this point to child configuration translators @Override public Map<String, Reportable> getSubMetrics() { return ReportUtil.noChildren(); } /** * {@inheritDoc} This always just forwards to the parent. */ @Override synchronized public void loadConfigFile(String file) throws IOException { parentMan.loadConfigFile(file); refreshAll(); } /** * {@inheritDoc} This always just forwards to the parent. */ @Override synchronized public void saveConfigFile(String file) throws IOException { parentMan.saveConfigFile(file); } /** * Returns the self configurations. */ @Override synchronized public Map<String, FlumeConfigData> getAllConfigs() { return parentMan.getAllConfigs(); } /** * Returns the translations of all configuraitons */ synchronized public Map<String, FlumeConfigData> getTranslatedConfigs() { return selfMan.getAllConfigs(); } /** * This reads a configuration and the sets it again. This updates the version * stamp and forces nodes to update their configurations. * * Since this manager intercepts the logical node configuration and writes the * user specified node to a different value, we actually read the user * specified source-sink pair and then use this manager's setConfig method to * include the autogenerated source-sink pair. */ synchronized public void refresh(String logicalNode) throws IOException { FlumeConfigData fcd = parentMan.getConfig(logicalNode); if (fcd == null) { throw new IOException("original " + logicalNode + " not found"); } try { setConfig(logicalNode, fcd.getFlowID(), fcd.getSourceConfig(), fcd.getSinkConfig()); } catch (FlumeSpecException e) { throw new IOException(e); } } /** * This reads a configuration and updates the version stamp only if the new * configuration is different from the previous configuration. */ synchronized public void updateAll() throws IOException { parentMan.updateAll(); Map<String, FlumeConfigData> updates = new HashMap<String, FlumeConfigData>(); for (Entry<String, FlumeConfigData> ent : parentMan.getTranslatedConfigs().entrySet()) { String node = ent.getKey(); // get the original name FlumeConfigData fcd = ent.getValue(); String src = fcd.getSourceConfig(); String snk = fcd.getSinkConfig(); String xsnk, xsrc; try { xsnk = translateSink(node, snk); xsrc = translateSource(node, src); FlumeConfigData selfData = selfMan.getConfig(node); if (selfData != null && xsnk.equals(selfData.getSinkConfig()) && xsrc.equals(selfData.getSourceConfig())) { // same as before? do nothing LOG.debug("xsnk==snk = " + xsnk); LOG.debug("xsrc==src = " + xsrc); continue; } FlumeConfigData xfcd = new FlumeConfigData(fcd); xfcd.setSourceConfig(xsrc); xfcd.setSinkConfig(xsnk); updates.put(node, xfcd); } catch (FlumeSpecException e) { LOG.error("Internal Error: " + e.getLocalizedMessage(), e); throw new IOException("Internal Error: " + e.getMessage()); } } selfMan.setBulkConfig(updates); } /** * This reads a configuration and the sets it again. This updates the version * stamp and forces nodes to update their configurations. * * Since this manager intercepts the logical node configuration and writes the * user specified node to a different value, we actually read the user * specified source-sink pair and then use this manager's setConfig method to * include the autogenerated source-sink pair. */ @Override synchronized public void refreshAll() throws IOException { parentMan.refreshAll(); Map<String, FlumeConfigData> updates = new HashMap<String, FlumeConfigData>(); for (Entry<String, FlumeConfigData> ent : parentMan.getTranslatedConfigs().entrySet()) { String node = ent.getKey(); // get the original name FlumeConfigData fcd = ent.getValue(); String src = fcd.getSourceConfig(); String snk = fcd.getSinkConfig(); String xsnk, xsrc; try { xsnk = translateSink(node, snk); xsrc = translateSource(node, src); FlumeConfigData xfcd = new FlumeConfigData(fcd); xfcd.setSinkConfig(xsnk); xfcd.setSourceConfig(xsrc); updates.put(node, xfcd); } catch (FlumeSpecException e) { LOG.error("Internal Error: " + e.getLocalizedMessage(), e); throw new IOException("Internal Error: " + e.getMessage()); } } selfMan.setBulkConfig(updates); } /** * Updates both the parent and self managers with the set of configurations. */ @Override synchronized public void setBulkConfig(Map<String, FlumeConfigData> configs) throws IOException { Map<String, FlumeConfigData> updates = new HashMap<String, FlumeConfigData>(); Map<String, FlumeConfigData> selfupdates = new HashMap<String, FlumeConfigData>(); for (Entry<String, FlumeConfigData> ent : configs.entrySet()) { String node = ent.getKey(); // get the original name String src = ent.getValue().getSourceConfig(); String snk = ent.getValue().getSinkConfig(); String xsnk, xsrc; try { xsnk = translateSink(node, snk); xsrc = translateSource(node, src); if (selfMan != parentMan) { FlumeConfigData fcd = new FlumeConfigData(ent.getValue()); updates.put(node, fcd); } FlumeConfigData xfcd = new FlumeConfigData(ent.getValue()); xfcd.setSinkConfig(xsnk); xfcd.setSourceConfig(xsrc); selfupdates.put(node, xfcd); } catch (FlumeSpecException e) { LOG.error("Internal Error: " + e.getLocalizedMessage(), e); throw new IOException("Internal Error: " + e.getMessage()); } } parentMan.setBulkConfig(updates); updateAll(); } /** * Remove the logical node. */ @Override synchronized public void removeLogicalNode(String logicNode) throws IOException { // only remove once if parent == self if (parentMan != selfMan) { parentMan.removeLogicalNode(logicNode); } selfMan.removeLogicalNode(logicNode); try { updateAll(); } catch (IOException e) { LOG.error("Error when removing logical node " + logicNode, e); } } /** * Start the sub managers. */ @Override synchronized public void start() throws IOException { // if parent == self, only start once. Preconditions.checkNotNull(this.parentMan, "Trying to start with null cfgMan"); if (parentMan != selfMan) { parentMan.start(); } selfMan.start(); updateAll(); } /** * Stop the sub managers. */ @Override synchronized public void stop() throws IOException { // if parent == self, only stop once. Preconditions.checkNotNull(this.parentMan, "Trying to stop with null cfgMan"); if (parentMan != selfMan) { parentMan.stop(); } selfMan.stop(); } // ////////////////////////////////////////////////////////////////////////////// // TODO decouple mapping from logical node configuration. Currently, // only forward node mappings calls to parent. /** * {@inheritDoc} */ @Override synchronized public Multimap<String, String> getLogicalNodeMap() { return parentMan.getLogicalNodeMap(); } /** * {@inheritDoc} */ synchronized public List<String> getLogicalNode(String physNode) { return parentMan.getLogicalNode(physNode); } /** * {@inheritDoc} */ synchronized public Map<String, Integer> getChokeMap(String physNode) { return parentMan.getChokeMap(physNode); } /** * {@inheritDoc} */ @Override synchronized public boolean addLogicalNode(String physNode, String logicNode) { boolean result; result = false; if (!getLogicalNodeMap().containsValue(logicNode)) { result = parentMan.addLogicalNode(physNode, logicNode); } try { updateAll(); } catch (IOException e) { LOG.error("Error when mapping logical->physical node" + logicNode + "->" + physNode, e); } return result; } /** * {@inheritDoc} */ @Override synchronized public void addChokeLimit(String physNode, String chokeID, int limit) { parentMan.addChokeLimit(physNode, chokeID, limit); } /** * {@inheritDoc} */ @Override synchronized public String getPhysicalNode(String logicalNode) { return parentMan.getPhysicalNode(logicalNode); } /** * {@inheritDoc} */ @Override synchronized public void unmapLogicalNode(String physNode, String logicNode) { parentMan.unmapLogicalNode(physNode, logicNode); try { updateAll(); } catch (IOException e) { LOG.error("Error when unmapping logical node " + e.getMessage(), e); } } /** * {@inheritDoc} */ @Override synchronized public void unmapAllLogicalNodes() throws IOException { // mapping is only on parent. parentMan.unmapAllLogicalNodes(); try { updateAll(); } catch (IOException e) { LOG.error("Error when unmapping all logical nodes" + e.getMessage(), e); } } /** * {@inheritDoc} */ @Override public String toString() { return getTranslatedConfigs().toString(); } }