Java tutorial
/* * Copyright 2008-2013 LinkedIn, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of * the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. */ package voldemort.server; import static voldemort.utils.Utils.croak; import java.io.File; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import org.apache.commons.io.IOUtils; import org.apache.log4j.Logger; import voldemort.VoldemortApplicationException; import voldemort.VoldemortException; import voldemort.annotations.jmx.JmxOperation; import voldemort.client.protocol.RequestFormatType; import voldemort.client.protocol.admin.AdminClient; import voldemort.cluster.Cluster; import voldemort.cluster.Node; import voldemort.common.service.AbstractService; import voldemort.common.service.SchedulerService; import voldemort.common.service.ServiceType; import voldemort.common.service.VoldemortService; import voldemort.rest.server.RestService; import voldemort.server.gossip.GossipService; import voldemort.server.http.HttpService; import voldemort.server.jmx.JmxService; import voldemort.server.niosocket.NioSocketService; import voldemort.server.protocol.ClientRequestHandlerFactory; import voldemort.server.protocol.RequestHandlerFactory; import voldemort.server.protocol.SocketRequestHandlerFactory; import voldemort.server.protocol.admin.AsyncOperationService; import voldemort.server.rebalance.Rebalancer; import voldemort.server.rebalance.RebalancerService; import voldemort.server.socket.SocketService; import voldemort.server.storage.StorageService; import voldemort.store.DisabledStoreException; import voldemort.store.StorageEngine; import voldemort.store.StoreCapabilityType; import voldemort.store.configuration.ConfigurationStorageEngine; import voldemort.store.metadata.MetadataStore; import voldemort.store.readonly.ReadOnlyStorageEngine; import voldemort.store.readonly.StoreVersionManager; import voldemort.store.readonly.swapper.FailedFetchLock; import voldemort.utils.ByteArray; import voldemort.utils.JNAUtils; import voldemort.utils.Props; import voldemort.utils.SystemTime; import voldemort.utils.Utils; import voldemort.versioning.VectorClock; import voldemort.versioning.Versioned; import voldemort.xml.ClusterMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; /** * This is the main server, it bootstraps all the services. * * It can be embedded or run directly via it's main method. * * */ public class VoldemortServer extends AbstractService { private static final Logger logger = Logger.getLogger(VoldemortServer.class.getName()); public static final long DEFAULT_PUSHER_POLL_MS = 60 * 1000; private final static int ASYNC_REQUEST_CACHE_SIZE = 64; private Node identityNode; private final List<VoldemortService> basicServices; private final StoreRepository storeRepository; private final VoldemortConfig voldemortConfig; private final MetadataStore metadata; private List<VoldemortService> onlineServices; private AsyncOperationService asyncService; private StorageService storageService; private JmxService jmxService; private VoldemortServer(VoldemortConfig config, MetadataStore metadataStore) { super(ServiceType.VOLDEMORT); this.voldemortConfig = config; this.setupSSLProvider(); this.metadata = metadataStore; this.storeRepository = new StoreRepository(config.isJmxEnabled()); // Update the config with right node Id this.refreshNodeIdFromMetadata(); this.checkHostName(); this.validateRestServiceConfiguration(); this.basicServices = createBasicServices(); createOnlineServices(); } public static int computeNodeId(VoldemortConfig config, Cluster cluster) { HostMatcher matcher = config.getNodeIdImplementation(); return NodeIdUtils.findNodeId(cluster, matcher); } public void validateNodeId() { if (voldemortConfig.getNodeId() != metadata.getNodeId()) { throw new VoldemortException("Voldemort Config Node Id " + voldemortConfig.getNodeId() + " does not match with metadata store node Id " + metadata.getNodeId()); } validateNodeId(voldemortConfig, metadata.getCluster()); } public static void validateNodeId(VoldemortConfig config, Cluster cluster) { if (config.isValidateNodeId() || config.isEnableNodeIdDetection()) { HostMatcher matcher = config.getNodeIdImplementation(); NodeIdUtils.validateNodeId(cluster, matcher, config.getNodeId()); } else { logger.info("Node id Validation is disabled in the config."); } } public static int getNodeId(VoldemortConfig config, Cluster cluster) { int configNodeId = config.getNodeId(); if (configNodeId >= 0) { return configNodeId; } if (!config.isEnableNodeIdDetection()) { // Node Id is missing and auto detection is disabled, error out. throw new VoldemortException( VoldemortConfig.NODE_ID + " is a required property of the Voldemort Server"); } return computeNodeId(config, cluster); } public void refreshNodeIdFromMetadata() { int nodeId = this.metadata.getNodeId(); voldemortConfig.setNodeId(nodeId); validateNodeId(); Node oldNode = this.identityNode; this.identityNode = metadata.getCluster().getNodeById(nodeId); if (oldNode != null) { if (oldNode.getSocketPort() != this.identityNode.getSocketPort() || oldNode.getAdminPort() != this.identityNode.getAdminPort()) { throw new VoldemortApplicationException("Node Id update, changes the Socket And Or Admin Port. " + "The Server will be in an inconsistent state, until the next restart. Old State " + oldNode.getStateString() + "New State " + this.identityNode.getStateString()); } } } public void handleClusterUpdate() { if (!voldemortConfig.isEnableNodeIdDetection()) { logger.info("Auto detection is disabled, returning"); return; } int nodeId = computeNodeId(voldemortConfig, metadata.getCluster()); // Put reInitializes the node Id as required. metadata.put(MetadataStore.NODE_ID_KEY, new Integer(nodeId)); refreshNodeIdFromMetadata(); } private static MetadataStore createMetadataFromConfig(VoldemortConfig voldemortConfig) { MetadataStore metadataStore = MetadataStore .readFromDirectory(new File(voldemortConfig.getMetadataDirectory())); int nodeId = getNodeId(voldemortConfig, metadataStore.getCluster()); metadataStore.initNodeId(nodeId); return metadataStore; } private static MetadataStore getTestMetadataStore(VoldemortConfig voldemortConfig, Cluster cluster) { ConfigurationStorageEngine metadataInnerEngine = new ConfigurationStorageEngine("metadata-config-store", voldemortConfig.getMetadataDirectory()); List<Versioned<String>> clusterXmlValue = metadataInnerEngine.get(MetadataStore.CLUSTER_KEY, null); VectorClock version = null; if (clusterXmlValue.size() <= 0) { version = new VectorClock(); } else { version = (VectorClock) clusterXmlValue.get(0).getVersion(); } int nodeId = getNodeId(voldemortConfig, cluster); version.incrementVersion(nodeId, System.currentTimeMillis()); metadataInnerEngine.put(MetadataStore.CLUSTER_KEY, new Versioned<String>(new ClusterMapper().writeCluster(cluster), version), null); return MetadataStore.createInMemoryMetadataStore(metadataInnerEngine, nodeId); } public VoldemortServer(VoldemortConfig config) { this(config, createMetadataFromConfig(config)); } /** * Constructor is used exclusively by tests. I.e., this is not a code path * that is exercised in production. * * @param config * @param cluster */ public VoldemortServer(VoldemortConfig config, Cluster cluster) { this(config, getTestMetadataStore(config, cluster)); } private void setupSSLProvider() { if (voldemortConfig.isBouncyCastleEnabled()) { // This is just a one line method, but using a separate class to // avoid loading the BouncyCastle. This will enable the // VoldemortServer to run without BouncyCastle in the class path // unless enabled explicitly. SetupSSLProvider.useBouncyCastle(); } } public AsyncOperationService getAsyncRunner() { return asyncService; } /** * Compare the configured hostname with all the ip addresses and hostnames * for the server node, and log a warning if there is a mismatch. * */ // TODO: VoldemortServer should throw exception if cluster xml, node id, and // server's state are not all mutually consistent. // // "I attempted to do this in the past. In practice its hard since the // hostname strings returned may not exactly match what's in cluster.xml // (ela4-app0000.prod vs ela4-app0000.prod.linkedin.com). And for folks // running with multiple interfaces and stuff in the open source world, not // sure how it would fan out.. // // I am in favour of doing this though.. May be implement a server config, // "strict.hostname.check.on.startup" which is false by default and true for // our environments and our SRE makes sure there is an exact match?" -- // VChandar // // "Strict host name doesn't work? We can always trim the rest before the comparison." // -- LGao private void checkHostName() { try { HashSet<String> ipAddrList = new HashSet<String>(); InetAddress localhost = InetAddress.getLocalHost(); InetAddress[] serverAddrs = InetAddress.getAllByName(localhost.getCanonicalHostName()); ipAddrList.add("localhost"); if (serverAddrs != null && serverAddrs.length > 0) { for (InetAddress addr : serverAddrs) { if (addr.getHostName() != null) ipAddrList.add(addr.getHostName()); if (addr.getHostAddress() != null) ipAddrList.add(addr.getHostAddress()); if (addr.getCanonicalHostName() != null) ipAddrList.add(addr.getCanonicalHostName()); } } if (!ipAddrList.contains(this.identityNode.getHost())) { logger.info("List of all IPs & Hostnames for the current node:" + ipAddrList); logger.info("Configured hostname [" + this.identityNode.getHost() + "] does not seem to match current node."); } } catch (UnknownHostException uhe) { logger.warn("Unable to obtain IP information for current node", uhe); } catch (SecurityException se) { logger.warn("Security Manager does not permit obtaining IP Information", se); } } /** * To start Rest Service two parameters need to be set: 1) set * "enable.rest=true" in server.properties 2) set "<rest-port>" in * cluster.xml. If rest Service is enabled without setting <rest-port>, the * system exits with an error log. */ private void validateRestServiceConfiguration() { boolean isRestEnabled = voldemortConfig.isRestServiceEnabled(); boolean isRestPortDefined = (identityNode.getRestPort() != -1) ? true : false; if (isRestEnabled != isRestPortDefined) { if (isRestEnabled) { String errorMessage = "Rest Service is enabled without defining \"rest-port\" in cluster.xml . " + this.identityNode.getStateString(); logger.error(errorMessage); throw new VoldemortApplicationException(errorMessage); } else { logger.warn("\"rest-port\" is defined in cluster.xml but Rest service is not enabled."); } } } public void createOnlineServices() { onlineServices = Lists.newArrayList(); if (voldemortConfig.isHttpServerEnabled()) { /* * TODO: Get rid of HTTP Service. */ HttpService httpService = new HttpService(this, storageService, storeRepository, RequestFormatType.VOLDEMORT_V1, voldemortConfig.getMaxThreads(), identityNode.getHttpPort()); onlineServices.add(httpService); } if (voldemortConfig.isRestServiceEnabled()) { RestService restService = new RestService(voldemortConfig, identityNode.getRestPort(), storeRepository, identityNode.getZoneId(), metadata.getStoreDefList()); onlineServices.add(restService); } if (voldemortConfig.isSocketServerEnabled()) { RequestHandlerFactory clientRequestHandlerFactory = new ClientRequestHandlerFactory( this.storeRepository); if (voldemortConfig.getUseNioConnector()) { logger.info("Using NIO Connector."); NioSocketService nioSocketService = new NioSocketService(clientRequestHandlerFactory, identityNode.getSocketPort(), voldemortConfig.getSocketBufferSize(), voldemortConfig.isNioConnectorKeepAlive(), voldemortConfig.getNioConnectorSelectors(), "nio-socket-server", voldemortConfig.isJmxEnabled(), voldemortConfig.getNioAcceptorBacklog(), voldemortConfig.getNioSelectorMaxHeartBeatTimeMs()); onlineServices.add(nioSocketService); } else { logger.info("Using BIO Connector."); SocketService socketService = new SocketService(clientRequestHandlerFactory, identityNode.getSocketPort(), voldemortConfig.getCoreThreads(), voldemortConfig.getMaxThreads(), voldemortConfig.getSocketBufferSize(), "socket-server", voldemortConfig.isJmxEnabled()); onlineServices.add(socketService); } } } private List<VoldemortService> createBasicServices() { /* Services are given in the order they must be started */ List<VoldemortService> services = new ArrayList<VoldemortService>(); SchedulerService scheduler = new SchedulerService(voldemortConfig.getSchedulerThreads(), SystemTime.INSTANCE, voldemortConfig.canInterruptService()); storageService = new StorageService(storeRepository, metadata, scheduler, voldemortConfig); asyncService = new AsyncOperationService(scheduler, ASYNC_REQUEST_CACHE_SIZE); jmxService = null; services.add(storageService); services.add(scheduler); services.add(asyncService); if (voldemortConfig.isAdminServerEnabled()) { Rebalancer rebalancer = null; if (voldemortConfig.isEnableRebalanceService()) { RebalancerService rebalancerService = new RebalancerService(storeRepository, metadata, voldemortConfig, asyncService, scheduler); services.add(rebalancerService); rebalancer = rebalancerService.getRebalancer(); } SocketRequestHandlerFactory adminRequestHandlerFactory = new SocketRequestHandlerFactory(storageService, this.storeRepository, this.metadata, this.voldemortConfig, this.asyncService, scheduler, rebalancer, this); if (voldemortConfig.getUseNioConnector()) { logger.info("Using NIO Connector for Admin Service."); services.add(new NioSocketService(adminRequestHandlerFactory, identityNode.getAdminPort(), voldemortConfig.getAdminSocketBufferSize(), voldemortConfig.isNioAdminConnectorKeepAlive(), voldemortConfig.getNioAdminConnectorSelectors(), "admin-server", voldemortConfig.isJmxEnabled(), voldemortConfig.getNioAcceptorBacklog(), voldemortConfig.getNioSelectorMaxHeartBeatTimeMs())); } else { logger.info("Using BIO Connector for Admin Service."); services.add(new SocketService(adminRequestHandlerFactory, identityNode.getAdminPort(), voldemortConfig.getAdminCoreThreads(), voldemortConfig.getAdminMaxThreads(), voldemortConfig.getAdminSocketBufferSize(), "admin-server", voldemortConfig.isJmxEnabled())); } } if (voldemortConfig.isGossipEnabled()) { services.add(new GossipService(this.metadata, scheduler, voldemortConfig)); } if (voldemortConfig.isJmxEnabled()) { jmxService = new JmxService(this, this.metadata.getCluster(), storeRepository, services); services.add(jmxService); } return ImmutableList.copyOf(services); } private void startOnlineServices() { if (jmxService != null) { jmxService.registerServices(onlineServices); } for (VoldemortService service : onlineServices) { service.start(); } } private List<VoldemortException> stopOnlineServices() { List<VoldemortException> exceptions = Lists.newArrayList(); for (VoldemortService service : Utils.reversed(onlineServices)) { try { service.stop(); } catch (VoldemortException e) { exceptions.add(e); logger.error(e); } } if (jmxService != null) { jmxService.unregisterServices(onlineServices); } return exceptions; } @Override protected void startInner() throws VoldemortException { // lock down jvm heap JNAUtils.tryMlockall(); logger.info("Starting " + basicServices.size() + " services."); long start = System.currentTimeMillis(); boolean goOnline; if (getMetadataStore().getServerStateUnlocked() == MetadataStore.VoldemortState.OFFLINE_SERVER) goOnline = false; else goOnline = true; for (VoldemortService service : basicServices) { try { service.start(); } catch (DisabledStoreException e) { logger.error("Got a DisabledStoreException from " + service.getType().getDisplayName(), e); goOnline = false; } } if (goOnline) { startOnlineServices(); } else { goOffline(); } long end = System.currentTimeMillis(); logger.info("Startup completed in " + (end - start) + " ms."); } /** * Attempt to shutdown the server. As much shutdown as possible will be * completed, even if intermediate errors are encountered. * * @throws VoldemortException */ @Override protected void stopInner() throws VoldemortException { List<VoldemortException> exceptions = new ArrayList<VoldemortException>(); logger.info("Stopping services:" + getIdentityNode().getId()); /* Stop in reverse order */ exceptions.addAll(stopOnlineServices()); for (VoldemortService service : Utils.reversed(basicServices)) { try { service.stop(); } catch (VoldemortException e) { exceptions.add(e); logger.error(e); } } logger.info("All services stopped for Node:" + getIdentityNode().getId()); if (exceptions.size() > 0) throw exceptions.get(0); // release lock of jvm heap JNAUtils.tryMunlockall(); } public static void main(String[] args) throws Exception { VoldemortConfig config = null; try { if (args.length == 0) config = VoldemortConfig.loadFromEnvironmentVariable(); else if (args.length == 1) config = VoldemortConfig.loadFromVoldemortHome(args[0]); else if (args.length == 2) config = VoldemortConfig.loadFromVoldemortHome(args[0], args[1]); else croak("USAGE: java " + VoldemortServer.class.getName() + " [voldemort_home_dir] [voldemort_config_dir]"); } catch (Exception e) { logger.error("Error while loading configuration", e); Utils.croak("Error while loading configuration. Will exit."); } final VoldemortServer server = new VoldemortServer(config); if (!server.isStarted()) server.start(); // add a shutdown hook to stop the server Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { if (server.isStarted()) server.stop(); } }); } public Node getIdentityNode() { return this.identityNode; } public List<VoldemortService> getServices() { return basicServices; } public VoldemortService getService(ServiceType type) { for (VoldemortService service : basicServices) if (service.getType().equals(type)) return service; throw new IllegalStateException(type.getDisplayName() + " has not been initialized."); } public VoldemortConfig getVoldemortConfig() { return this.voldemortConfig; } public StoreRepository getStoreRepository() { return this.storeRepository; } public MetadataStore getMetadataStore() { return metadata; } @JmxOperation(description = "force restore data from replication") public void restoreDataFromReplication(int numberOfParallelTransfers) { AdminClient adminClient = AdminClient.createTempAdminClient(voldemortConfig, metadata.getCluster(), numberOfParallelTransfers * 2); try { adminClient.restoreOps.restoreDataFromReplications(metadata.getNodeId(), numberOfParallelTransfers); } finally { adminClient.close(); } } public void goOffline() { getMetadataStore().setOfflineState(true); stopOnlineServices(); } public void goOnline() { ReadOnlyStoreStatusValidation validation = validateReadOnlyStoreStatusBeforeGoingOnline(); if (validation.readyToGoOnline) { getMetadataStore().setOfflineState(false); createOnlineServices(); startOnlineServices(); } if (validation.e != null) { throw new VoldemortException("Problem while going online!", validation.e); } } private class ReadOnlyStoreStatusValidation { /** Whether the server should go online (i.e.: it has no disabled stores) */ private final boolean readyToGoOnline; /** Whether the admin operation should return an error (this is orthogonal to whether the server went online or not) */ private final Exception e; ReadOnlyStoreStatusValidation(boolean readyToGoOnline, Exception e) { this.readyToGoOnline = readyToGoOnline; this.e = e; } } private ReadOnlyStoreStatusValidation validateReadOnlyStoreStatusBeforeGoingOnline() { List<StorageEngine<ByteArray, byte[], byte[]>> storageEngines = storageService.getStoreRepository() .getStorageEnginesByClass(ReadOnlyStorageEngine.class); if (storageEngines.isEmpty()) { logger.debug("There are no Read-Only stores on this node."); return new ReadOnlyStoreStatusValidation(true, null); } else { List<String> storesWithDisabledVersions = Lists.newArrayList(); for (StorageEngine storageEngine : storageEngines) { StoreVersionManager storeVersionManager = (StoreVersionManager) storageEngine .getCapability(StoreCapabilityType.DISABLE_STORE_VERSION); if (storeVersionManager.hasAnyDisabledVersion()) { storesWithDisabledVersions.add(storageEngine.getName()); } } if (storesWithDisabledVersions.isEmpty()) { if (voldemortConfig.getHighAvailabilityStateAutoCleanUp()) { logger.info(VoldemortConfig.PUSH_HA_STATE_AUTO_CLEANUP + "=true, so the server will attempt to delete the HA state for this node, if any."); FailedFetchLock failedFetchLock = null; try { failedFetchLock = FailedFetchLock.getLock(getVoldemortConfig(), new Props()); failedFetchLock.removeObsoleteStateForNode(getVoldemortConfig().getNodeId()); logger.info("Successfully ensured that the BnP HA shared state is cleared for this node."); } catch (ClassNotFoundException e) { return new ReadOnlyStoreStatusValidation(true, new VoldemortException("Failed to find FailedFetchLock class!", e)); } catch (Exception e) { return new ReadOnlyStoreStatusValidation(true, new VoldemortException("Exception while trying to remove obsolete HA state!", e)); } finally { IOUtils.closeQuietly(failedFetchLock); } } else { logger.info(VoldemortConfig.PUSH_HA_STATE_AUTO_CLEANUP + "=false, so the server will NOT attempt to delete the HA state for this node, if any."); } logger.info("No Read-Only stores are disabled. Going online as planned."); return new ReadOnlyStoreStatusValidation(true, null); } else { // OMG, there are disabled stores! StringBuilder stringBuilder = new StringBuilder(); stringBuilder.append( "Cannot go online, because the following Read-Only stores have some disabled version(s): "); boolean firstItem = true; for (String storeName : storesWithDisabledVersions) { if (firstItem) { firstItem = false; } else { stringBuilder.append(", "); } stringBuilder.append(storeName); } return new ReadOnlyStoreStatusValidation(false, new VoldemortException(stringBuilder.toString())); } } } }