Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more contributor license * agreements. See the NOTICE file distributed with this work for additional information regarding * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. You may obtain a * copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package org.apache.geode.distributed.internal; import static org.apache.geode.distributed.ConfigurationProperties.BIND_ADDRESS; import static org.apache.geode.distributed.ConfigurationProperties.CACHE_XML_FILE; import static org.apache.geode.distributed.ConfigurationProperties.LOCATORS; import java.io.File; import java.io.IOException; import java.net.ConnectException; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Properties; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.lang.StringUtils; import org.apache.logging.log4j.Logger; import org.apache.geode.CancelException; import org.apache.geode.cache.CacheFactory; import org.apache.geode.cache.GemFireCache; import org.apache.geode.cache.client.internal.locator.ClientConnectionRequest; import org.apache.geode.cache.client.internal.locator.ClientReplacementRequest; import org.apache.geode.cache.client.internal.locator.GetAllServersRequest; import org.apache.geode.cache.client.internal.locator.LocatorListRequest; import org.apache.geode.cache.client.internal.locator.LocatorStatusRequest; import org.apache.geode.cache.client.internal.locator.QueueConnectionRequest; import org.apache.geode.cache.client.internal.locator.wan.LocatorMembershipListener; import org.apache.geode.distributed.DistributedSystem; import org.apache.geode.distributed.Locator; import org.apache.geode.distributed.LockServiceDestroyedException; import org.apache.geode.distributed.internal.InternalDistributedSystem.ConnectListener; import org.apache.geode.distributed.internal.InternalDistributedSystem.DisconnectListener; import org.apache.geode.distributed.internal.membership.MemberFactory; import org.apache.geode.distributed.internal.membership.QuorumChecker; import org.apache.geode.distributed.internal.membership.gms.NetLocator; import org.apache.geode.distributed.internal.membership.gms.locator.PeerLocatorRequest; import org.apache.geode.distributed.internal.tcpserver.LocatorCancelException; import org.apache.geode.distributed.internal.tcpserver.TcpClient; import org.apache.geode.distributed.internal.tcpserver.TcpHandler; import org.apache.geode.distributed.internal.tcpserver.TcpServer; import org.apache.geode.internal.admin.remote.DistributionLocatorId; import org.apache.geode.internal.cache.GemFireCacheImpl; import org.apache.geode.internal.cache.InternalCache; import org.apache.geode.internal.cache.tier.sockets.ClientProtocolService; import org.apache.geode.internal.cache.tier.sockets.TcpServerFactory; import org.apache.geode.internal.cache.wan.WANServiceProvider; import org.apache.geode.internal.i18n.LocalizedStrings; import org.apache.geode.internal.logging.InternalLogWriter; import org.apache.geode.internal.logging.LogService; import org.apache.geode.internal.logging.LogWriterFactory; import org.apache.geode.internal.logging.LoggingThreadGroup; import org.apache.geode.internal.logging.log4j.LocalizedMessage; import org.apache.geode.internal.logging.log4j.LogWriterAppenders; import org.apache.geode.internal.net.SocketCreator; import org.apache.geode.internal.net.SocketCreatorFactory; import org.apache.geode.management.internal.JmxManagerLocator; import org.apache.geode.management.internal.JmxManagerLocatorRequest; import org.apache.geode.management.internal.cli.CliUtil; import org.apache.geode.management.internal.configuration.domain.SharedConfigurationStatus; import org.apache.geode.management.internal.configuration.handlers.ConfigurationRequestHandler; import org.apache.geode.management.internal.configuration.handlers.SharedConfigurationStatusRequestHandler; import org.apache.geode.management.internal.configuration.messages.ConfigurationRequest; import org.apache.geode.management.internal.configuration.messages.SharedConfigurationStatusRequest; import org.apache.geode.management.internal.configuration.messages.SharedConfigurationStatusResponse; /** * Provides the implementation of a distribution {@code Locator} as well as internal-only * functionality. * <p> * This class has APIs that perform essentially three layers of services. At the bottom layer is the * JGroups location service. On top of that you can start a distributed system. And then on top of * that you can start server location services. * <p> * Server Location Service DistributedSystem Peer Location Service * <p> * The startLocator() methods provide a way to start all three services in one call. Otherwise, the * services can be started independently {@code locator = createLocator(); * locator.startPeerLocation(); locator.startDistributeSystem();} * * @since GemFire 4.0 */ public class InternalLocator extends Locator implements ConnectListener { private static final Logger logger = LogService.getLogger(); /** * system property name for forcing an locator distribution manager type */ public static final String FORCE_LOCATOR_DM_TYPE = "Locator.forceLocatorDMType"; /** * system property name for inhibiting DM banner */ public static final String INHIBIT_DM_BANNER = "Locator.inhibitDMBanner"; /** * system property name for forcing locators to be preferred as coordinators */ public static final String LOCATORS_PREFERRED_AS_COORDINATORS = DistributionConfig.GEMFIRE_PREFIX + "disable-floating-coordinator"; /** * The tcp server responding to locator requests */ private final TcpServer server; /** * @since GemFire 5.7 */ private final PrimaryHandler handler; /** * The distributed system owned by this locator, if any. Note that if a ds already exists because * the locator is being colocated in a normal member this field will be null. */ private InternalDistributedSystem myDs; /** * The cache owned by this locator, if any. Note that if a cache already exists because the * locator is being colocated in a normal member this field will be null. */ private InternalCache myCache; /** * locator state file */ private File stateFile; /** * product use logging */ private ProductUseLog productUseLog; private boolean peerLocator; private ServerLocator serverLocator; protected volatile LocatorStats stats; private Properties env; /** * the TcpHandler used for peer location */ private NetLocator locatorImpl; private DistributionConfigImpl config; private final LocatorMembershipListener locatorListener; private WanLocatorDiscoverer locatorDiscoverer; /** * whether the locator was stopped during forced-disconnect processing but a reconnect will occur */ private volatile boolean stoppedForReconnect; /** * whether the locator was stopped during forced-disconnect processing */ private volatile boolean forcedDisconnect; private final AtomicBoolean shutdownHandled = new AtomicBoolean(false); private ClusterConfigurationService sharedConfig; private volatile boolean isSharedConfigurationStarted = false; private volatile Thread restartThread; boolean isSharedConfigurationEnabled() { return this.config.getEnableClusterConfiguration(); } private boolean loadFromSharedConfigDir() { return this.config.getLoadClusterConfigFromDir(); } public boolean isSharedConfigurationRunning() { return this.sharedConfig != null && this.sharedConfig.getStatus() == SharedConfigurationStatus.RUNNING; } /** * the locator hosted by this JVM. As of 7.0 it is a singleton. * * GuardedBy must synchronize on locatorLock */ private static InternalLocator locator; private static final Object locatorLock = new Object(); // TODO: getLocator() overrides static method of a superclass public static InternalLocator getLocator() { // synchronize in order to fix #46336 (race condition in createLocator) synchronized (locatorLock) { return locator; } } // TODO: hasLocator() overrides static method of a superclass public static boolean hasLocator() { synchronized (locatorLock) { return locator != null; } } // TODO: return value of removeLocator is never used private static boolean removeLocator(InternalLocator locator) { if (locator == null) { return false; } synchronized (locatorLock) { LogWriterAppenders.stop(LogWriterAppenders.Identifier.MAIN); LogWriterAppenders.stop(LogWriterAppenders.Identifier.SECURITY); LogWriterAppenders.destroy(LogWriterAppenders.Identifier.MAIN); LogWriterAppenders.destroy(LogWriterAppenders.Identifier.SECURITY); if (locator != null && locator.equals(InternalLocator.locator)) { InternalLocator.locator = null; return true; } return false; } } public LocatorMembershipListener getlocatorMembershipListener() { return this.locatorListener; } /** * Create a locator that listens on a given port. This locator will not have peer or server * location services available until they are started by calling startServerLocation or * startPeerLocation on the locator object. * * @param port the tcp/ip port to listen on * @param logFile the file that log messages should be written to * @param stateFile the file that state should be read from / written to for recovery * @param logger a log writer that should be used (logFile parameter is ignored) * @param securityLogger the logger to be used for security related log messages * @param distributedSystemProperties optional properties to configure the distributed system * (e.g., mcast addr/port, other locators) * @param startDistributedSystem if true then this locator will also start its own ds */ public static InternalLocator createLocator(int port, File logFile, File stateFile, InternalLogWriter logger, InternalLogWriter securityLogger, InetAddress bindAddress, String hostnameForClients, Properties distributedSystemProperties, boolean startDistributedSystem) { synchronized (locatorLock) { if (hasLocator()) { throw new IllegalStateException( "A locator can not be created because one already exists in this JVM."); } InternalLocator locator = new InternalLocator(port, logFile, stateFile, logger, securityLogger, bindAddress, hostnameForClients, distributedSystemProperties, null, startDistributedSystem); InternalLocator.locator = locator; return locator; } } private static void setLocator(InternalLocator locator) { synchronized (locatorLock) { if (InternalLocator.locator != null && InternalLocator.locator != locator) { throw new IllegalStateException( "A locator can not be created because one already exists in this JVM."); } InternalLocator.locator = locator; } } /** * Creates a distribution locator that runs in this VM on the given port and bind address. * <p> * This is for internal use only as it does not create a distributed system unless told to do so. * * @param port the tcp/ip port to listen on * @param logFile the file that log messages should be written to * @param logger a log writer that should be used (logFile parameter is ignored) * @param securityLogger the logger to be used for security related log messages * @param startDistributedSystem if true, a distributed system is started * @param dsProperties optional properties to configure the distributed system (e.g., mcast * addr/port, other locators) * @param hostnameForClients the name to give to clients for connecting to this locator */ public static InternalLocator startLocator(int port, File logFile, File stateFile, InternalLogWriter logger, InternalLogWriter securityLogger, InetAddress bindAddress, boolean startDistributedSystem, Properties dsProperties, String hostnameForClients) throws IOException { System.setProperty(FORCE_LOCATOR_DM_TYPE, "true"); InternalLocator newLocator = null; boolean startedLocator = false; try { newLocator = createLocator(port, logFile, stateFile, logger, securityLogger, bindAddress, hostnameForClients, dsProperties, startDistributedSystem); // TODO:GEODE-1243: this.server is now a TcpServer and it should store or return its non-zero // port in a variable to use here try { newLocator.startPeerLocation(startDistributedSystem); if (startDistributedSystem) { try { // TODO:GEODE-1243: throws Exception if TcpServer still has zero for its locator port newLocator.startDistributedSystem(); } catch (RuntimeException e) { newLocator.stop(); throw e; } // fix bug #46324 final InternalDistributedSystem ids = newLocator.myDs; if (ids != null) { ids.getDistributionManager().addHostedLocators(ids.getDistributedMember(), getLocatorStrings(), newLocator.isSharedConfigurationEnabled()); } } } catch (final LocatorCancelException ignored) { newLocator.stop(); } InternalDistributedSystem sys = InternalDistributedSystem.getConnectedInstance(); if (sys != null) { try { newLocator.startServerLocation(sys); } catch (RuntimeException e) { newLocator.stop(); throw e; } } newLocator.endStartLocator(null); startedLocator = true; return newLocator; } finally { System.getProperties().remove(FORCE_LOCATOR_DM_TYPE); if (!startedLocator) { // fix for bug 46314 removeLocator(newLocator); } } } /*** * Determines if this VM is a locator which must ignore a shutdown. * * @return true if this VM is a locator which should ignore a shutdown , false if it is a normal * member. */ public static boolean isDedicatedLocator() { InternalLocator internalLocator = getLocator(); if (internalLocator == null) { return false; } InternalDistributedSystem ids = internalLocator.myDs; if (ids == null) { return false; } DM dm = ids.getDistributionManager(); if (dm.isLoner()) { return false; } DistributionManager distMgr = (DistributionManager) ids.getDistributionManager(); return distMgr.getDMType() == DistributionManager.LOCATOR_DM_TYPE; } /** * Creates a new {@code Locator} with the given port, log file, logger, and bind address. * * @param port the tcp/ip port to listen on * @param logF the file that log messages should be written to * @param stateF the file that state should be read from / written to for recovery * @param logWriter a log writer that should be used (logFile parameter is ignored) * @param securityLogWriter the log writer to be used for security related log messages * @param hostnameForClients the name to give to clients for connecting to this locator * @param distributedSystemProperties optional properties to configure the distributed system * (e.g., mcast addr/port, other locators) * @param cfg the config if being called from a distributed system; otherwise null. * @param startDistributedSystem if true locator will start its own distributed system */ private InternalLocator(int port, File logF, File stateF, InternalLogWriter logWriter, // LOG: 3 non-null sources: GemFireDistributionLocator, InternalDistributedSystem, // LocatorLauncher InternalLogWriter securityLogWriter, // LOG: 1 non-null source: GemFireDistributionLocator(same instance as logWriter), // InternalDistributedSystem InetAddress bindAddress, String hostnameForClients, Properties distributedSystemProperties, DistributionConfigImpl cfg, boolean startDistributedSystem) { // TODO: the following three assignments are already done in superclass this.logFile = logF; this.bindAddress = bindAddress; this.hostnameForClients = hostnameForClients; if (stateF == null) { this.stateFile = new File("locator" + port + "view.dat"); } else { this.stateFile = stateF; } File productUseFile = new File("locator" + port + "views.log"); this.productUseLog = new ProductUseLog(productUseFile); this.config = cfg; this.env = new Properties(); // set bind-address explicitly only if not wildcard and let any explicit // value in distributedSystemProperties take precedence (#46870) if (bindAddress != null && !bindAddress.isAnyLocalAddress()) { this.env.setProperty(BIND_ADDRESS, bindAddress.getHostAddress()); } if (distributedSystemProperties != null) { this.env.putAll(distributedSystemProperties); } this.env.setProperty(CACHE_XML_FILE, ""); // create a DC so that all of the lookup rules, gemfire.properties, etc, // are considered and we have a config object we can trust if (this.config == null) { this.config = new DistributionConfigImpl(this.env); this.env.clear(); this.env.putAll(this.config.getProps()); } final boolean hasLogFileButConfigDoesNot = this.logFile != null && this.config.getLogFile().toString().equals(DistributionConfig.DEFAULT_LOG_FILE.toString()); if (logWriter == null && hasLogFileButConfigDoesNot) { // LOG: this is(was) a hack for when logFile and config don't match -- if config specifies a // different log-file things will break! this.config.unsafeSetLogFile(this.logFile); } // LOG: create LogWriterAppenders (these are closed at shutdown) final boolean hasLogFile = this.config.getLogFile() != null && !this.config.getLogFile().equals(new File("")); final boolean hasSecurityLogFile = this.config.getSecurityLogFile() != null && !this.config.getSecurityLogFile().equals(new File("")); LogService.configureLoggers(hasLogFile, hasSecurityLogFile); if (hasLogFile || hasSecurityLogFile) { if (hasLogFile) { // if log-file then create logWriterAppender LogWriterAppenders.getOrCreateAppender(LogWriterAppenders.Identifier.MAIN, true, false, this.config, !startDistributedSystem); } if (hasSecurityLogFile) { // if security-log-file then create securityLogWriterAppender LogWriterAppenders.getOrCreateAppender(LogWriterAppenders.Identifier.SECURITY, true, false, this.config, false); } // do not create a LogWriterAppender for security -- let it go through to logWriterAppender } // LOG: create LogWriters for GemFireTracer (or use whatever was passed in) if (logWriter == null) { logWriter = LogWriterFactory.createLogWriterLogger(false, false, this.config, false); if (logger.isDebugEnabled()) { logger.debug("LogWriter for locator is created."); } } if (securityLogWriter == null) { securityLogWriter = LogWriterFactory.createLogWriterLogger(false, true, this.config, false); logWriter.setLogWriterLevel(this.config.getSecurityLogLevel()); securityLogWriter.fine("SecurityLogWriter for locator is created."); } SocketCreatorFactory.setDistributionConfig(this.config); this.locatorListener = WANServiceProvider.createLocatorMembershipListener(); if (this.locatorListener != null) { // We defer setting the port until the handler is init'd - that way we'll have an actual port // in the case where we're starting with port = 0. this.locatorListener.setConfig(getConfig()); } this.handler = new PrimaryHandler(this, locatorListener); ThreadGroup group = LoggingThreadGroup.createThreadGroup("Distribution locators", logger); this.stats = new LocatorStats(); this.server = new TcpServerFactory().makeTcpServer(port, this.bindAddress, null, this.config, this.handler, new DelayedPoolStatHelper(), group, this.toString(), this); } // Reset the file names with the correct port number if startLocatorAndDS was called with port // number 0 public void resetInternalLocatorFileNamesWithCorrectPortNumber(int port) { this.stateFile = new File("locator" + port + "view.dat"); File productUseFile = new File("locator" + port + "views.log"); this.productUseLog = new ProductUseLog(productUseFile); } private void startTcpServer() throws IOException { logger.info(LocalizedMessage.create(LocalizedStrings.InternalLocator_STARTING_0, this)); this.server.start(); } public ClusterConfigurationService getSharedConfiguration() { return this.sharedConfig; } public DistributionConfigImpl getConfig() { return this.config; } public InternalCache getCache() { return myCache; } /** * Start peer location in this locator. If you plan on starting a distributed system later, this * method should be called first so that the distributed system can use this locator. * <p> * TODO: parameter withDS is never used * * @param withDS true if a distributed system has been or will be started * * @since GemFire 5.7 */ void startPeerLocation(boolean withDS) throws IOException { if (isPeerLocator()) { throw new IllegalStateException(LocalizedStrings.InternalLocator_PEER_LOCATION_IS_ALREADY_RUNNING_FOR_0 .toLocalizedString(this)); } logger.info(LocalizedMessage.create(LocalizedStrings.InternalLocator_STARTING_PEER_LOCATION_FOR_0, this)); String locatorsProp = this.config.getLocators(); // check for settings that would require only locators to hold the // coordinator - e.g., security and network-partition detection boolean locatorsAreCoordinators = false; boolean networkPartitionDetectionEnabled = this.config.getEnableNetworkPartitionDetection(); String securityUDPDHAlgo = this.config.getSecurityUDPDHAlgo(); if (networkPartitionDetectionEnabled) { locatorsAreCoordinators = true; } else { // check if security is enabled String prop = this.config.getSecurityPeerAuthInit(); locatorsAreCoordinators = prop != null && !prop.isEmpty(); if (!locatorsAreCoordinators) { locatorsAreCoordinators = Boolean.getBoolean(LOCATORS_PREFERRED_AS_COORDINATORS); } } this.locatorImpl = MemberFactory.newLocatorHandler(this.bindAddress, this.stateFile, locatorsProp, locatorsAreCoordinators, networkPartitionDetectionEnabled, this.stats, securityUDPDHAlgo); this.handler.addHandler(PeerLocatorRequest.class, this.locatorImpl); this.peerLocator = true; if (!this.server.isAlive()) { startTcpServer(); } } /** * @return the TcpHandler for peer to peer discovery */ public NetLocator getLocatorHandler() { return this.locatorImpl; } public PrimaryHandler getPrimaryHandler() { return this.handler; } /** * For backward-compatibility we retain this method * <p> * TODO: parameters peerLocator and serverLocator and b1 are never used * * @deprecated use a form of the method that does not have peerLocator/serverLocator parameters */ @Deprecated public static InternalLocator startLocator(int locatorPort, File logFile, File stateFile, InternalLogWriter logger, InternalLogWriter logger1, InetAddress addr, Properties dsProperties, boolean peerLocator, boolean serverLocator, String s, boolean b1) throws IOException { return startLocator(locatorPort, logFile, stateFile, logger, logger1, addr, true, dsProperties, s); } class SharedConfigurationRunnable implements Runnable { private final InternalLocator locator = InternalLocator.this; @Override public void run() { try { if (this.locator.sharedConfig == null) { // locator.sharedConfig will already be created in case of auto-reconnect this.locator.sharedConfig = new ClusterConfigurationService(locator.myCache); } this.locator.sharedConfig.initSharedConfiguration(this.locator.loadFromSharedConfigDir()); this.locator.installSharedConfigDistribution(); logger.info( "Cluster configuration service start up completed successfully and is now running ...."); } catch (CancelException | LockServiceDestroyedException e) { if (logger.isDebugEnabled()) { logger.debug("Cluster configuration start up was cancelled", e); } } catch (Exception e) { logger.error(e.getMessage(), e); } } } /** * Start a distributed system whose life cycle is managed by this locator. When the locator is * stopped, this distributed system will be disconnected. If a distributed system already exists, * this method will have no affect. * * @since GemFire 5.7 */ private void startDistributedSystem() throws UnknownHostException { InternalDistributedSystem existing = InternalDistributedSystem.getConnectedInstance(); if (existing != null) { // LOG: changed from config to info logger.info(LocalizedMessage .create(LocalizedStrings.InternalLocator_USING_EXISTING_DISTRIBUTED_SYSTEM__0, existing)); startCache(existing); } else { StringBuilder sb = new StringBuilder(100); if (this.bindAddress != null) { sb.append(this.bindAddress.getHostAddress()); } else { sb.append(SocketCreator.getLocalHost().getHostAddress()); } sb.append('[').append(getPort()).append(']'); String thisLocator = sb.toString(); if (this.peerLocator) { // append this locator to the locators list from the config properties // this.logger.config("ensuring that this locator is in the locators list"); boolean setLocatorsProp = false; String locatorsProp = this.config.getLocators(); if (StringUtils.isNotBlank(locatorsProp)) { if (!locatorsProp.contains(thisLocator)) { locatorsProp = locatorsProp + ',' + thisLocator; setLocatorsProp = true; } } else { locatorsProp = thisLocator; setLocatorsProp = true; } if (setLocatorsProp) { Properties updateEnv = new Properties(); updateEnv.setProperty(LOCATORS, locatorsProp); this.config.setApiProps(updateEnv); // fix for bug 41248 String propName = DistributionConfig.GEMFIRE_PREFIX + LOCATORS; if (System.getProperty(propName) != null) { System.setProperty(propName, locatorsProp); } } // No longer default mcast-port to zero. See 46277. } Properties connectEnv = new Properties(); // LogWriterAppender is now shared via that class // using a DistributionConfig earlier in this method connectEnv.put(DistributionConfig.DS_CONFIG_NAME, this.config); logger.info(LocalizedMessage.create(LocalizedStrings.InternalLocator_STARTING_DISTRIBUTED_SYSTEM)); this.myDs = (InternalDistributedSystem) DistributedSystem.connect(connectEnv); if (this.peerLocator) { this.locatorImpl.setMembershipManager(this.myDs.getDM().getMembershipManager()); } this.myDs.addDisconnectListener(new DisconnectListener() { @Override public void onDisconnect(InternalDistributedSystem sys) { stop(false, false, false); } }); startCache(myDs); logger.info( LocalizedMessage.create(LocalizedStrings.InternalLocator_LOCATOR_STARTED_ON__0, thisLocator)); myDs.setDependentLocator(this); } } private void startCache(DistributedSystem ds) { InternalCache internalCache = GemFireCacheImpl.getInstance(); if (internalCache == null) { logger.info("Creating cache for locator."); this.myCache = (InternalCache) new CacheFactory(ds.getProperties()).create(); internalCache = this.myCache; } else { logger.info("Using existing cache for locator."); ((InternalDistributedSystem) ds).handleResourceEvent(ResourceEvent.LOCATOR_START, this); } startJmxManagerLocationService(internalCache); startSharedConfigurationService(internalCache); } /** * End the initialization of the locator. This method should be called once the location services * and distributed system are started. * * @param distributedSystem The distributed system to use for the statistics. * * @since GemFire 5.7 */ void endStartLocator(InternalDistributedSystem distributedSystem) { this.env = null; if (distributedSystem == null) { distributedSystem = InternalDistributedSystem.getConnectedInstance(); } if (distributedSystem != null) { onConnect(distributedSystem); } else { InternalDistributedSystem.addConnectListener(this); } this.locatorDiscoverer = WANServiceProvider.createLocatorDiscoverer(); if (this.locatorDiscoverer != null) { this.locatorDiscoverer.discover(getPort(), this.config, this.locatorListener, this.hostnameForClients); } } /** * Start server location services in this locator. Server location can only be started once there * is a running distributed system. * * @param distributedSystem The distributed system which the server location services should use. * If null, the method will try to find an already connected distributed system. * * @since GemFire 5.7 */ void startServerLocation(InternalDistributedSystem distributedSystem) throws IOException { if (isServerLocator()) { throw new IllegalStateException( LocalizedStrings.InternalLocator_SERVER_LOCATION_IS_ALREADY_RUNNING_FOR_0 .toLocalizedString(this)); } logger.info(LocalizedMessage.create(LocalizedStrings.InternalLocator_STARTING_SERVER_LOCATION_FOR_0, this)); if (distributedSystem == null) { distributedSystem = InternalDistributedSystem.getConnectedInstance(); if (distributedSystem == null) { throw new IllegalStateException( LocalizedStrings.InternalLocator_SINCE_SERVER_LOCATION_IS_ENABLED_THE_DISTRIBUTED_SYSTEM_MUST_BE_CONNECTED .toLocalizedString()); } } this.productUseLog.monitorUse(distributedSystem); ServerLocator serverLocator = new ServerLocator(getPort(), this.bindAddress, this.hostnameForClients, this.logFile, this.productUseLog, getConfig().getName(), distributedSystem, this.stats); this.handler.addHandler(LocatorListRequest.class, serverLocator); this.handler.addHandler(ClientConnectionRequest.class, serverLocator); this.handler.addHandler(QueueConnectionRequest.class, serverLocator); this.handler.addHandler(ClientReplacementRequest.class, serverLocator); this.handler.addHandler(GetAllServersRequest.class, serverLocator); this.handler.addHandler(LocatorStatusRequest.class, serverLocator); this.serverLocator = serverLocator; if (!this.server.isAlive()) { startTcpServer(); } } /** * Stop this locator. */ @Override public void stop() { stop(false, false, true); } /** * Stop this locator * * @param stopForReconnect - stopping for distributed system reconnect * @param waitForDisconnect - wait up to 60 seconds for the locator to completely stop */ public void stop(boolean forcedDisconnect, boolean stopForReconnect, boolean waitForDisconnect) { final boolean isDebugEnabled = logger.isDebugEnabled(); this.stoppedForReconnect = stopForReconnect; this.forcedDisconnect = forcedDisconnect; if (this.server.isShuttingDown()) { // fix for bug 46156 // If we are already shutting down don't do all of this again. // But, give the server a bit of time to shut down so a new // locator can be created, if desired, when this method returns if (!stopForReconnect && waitForDisconnect) { long endOfWait = System.currentTimeMillis() + 60000; if (isDebugEnabled && this.server.isAlive()) { logger.debug("sleeping to wait for the locator server to shut down..."); } while (this.server.isAlive() && System.currentTimeMillis() < endOfWait) { try { Thread.sleep(500); } catch (InterruptedException ignored) { Thread.currentThread().interrupt(); return; } } if (isDebugEnabled) { if (this.server.isAlive()) { logger.debug( "60 seconds have elapsed waiting for the locator server to shut down - terminating wait and returning"); } else { logger.debug("the locator server has shut down"); } } } return; } if (this.locatorDiscoverer != null) { this.locatorDiscoverer.stop(); this.locatorDiscoverer = null; } if (this.server.isAlive()) { logger.info(LocalizedMessage.create(LocalizedStrings.InternalLocator_STOPPING__0, this)); try { new TcpClient().stop(this.bindAddress, getPort()); } catch (ConnectException ignore) { // must not be running } boolean interrupted = Thread.interrupted(); try { this.server.join(TcpServer.SHUTDOWN_WAIT_TIME * 1000 + 10000); } catch (InterruptedException ex) { interrupted = true; logger.warn(LocalizedMessage.create(LocalizedStrings.InternalLocator_INTERRUPTED_WHILE_STOPPING__0, this), ex); // Continue running -- doing our best to stop everything... } finally { if (interrupted) { Thread.currentThread().interrupt(); } } if (this.server.isAlive()) { logger.fatal(LocalizedMessage .create(LocalizedStrings.InternalLocator_COULD_NOT_STOP__0__IN_60_SECONDS, this)); } } removeLocator(this); handleShutdown(); logger.info(LocalizedMessage.create(LocalizedStrings.InternalLocator_0__IS_STOPPED, this)); if (this.stoppedForReconnect) { if (this.myDs != null) { launchRestartThread(); } } } /** * answers whether this locator is currently stopped */ public boolean isStopped() { return this.server == null || !this.server.isAlive(); } private void handleShutdown() { if (!this.shutdownHandled.compareAndSet(false, true)) { return; // already shutdown } this.productUseLog.close(); if (this.myDs != null) { this.myDs.setDependentLocator(null); } if (this.myCache != null && !this.stoppedForReconnect && !this.forcedDisconnect) { logger.info("Closing locator's cache"); try { this.myCache.close(); } catch (RuntimeException ex) { logger.info("Could not close locator's cache because: {}", ex.getMessage(), ex); } } if (this.stats != null) { this.stats.close(); } if (this.locatorListener != null) { this.locatorListener.clearLocatorInfo(); } this.isSharedConfigurationStarted = false; if (this.myDs != null && !this.forcedDisconnect) { if (this.myDs.isConnected()) { logger.info(LocalizedMessage .create(LocalizedStrings.InternalLocator_DISCONNECTING_DISTRIBUTED_SYSTEM_FOR_0, this)); this.myDs.disconnect(); } } } /** * Waits for a locator to be told to stop. * * @throws InterruptedException thrown if the thread is interrupted */ public void waitToStop() throws InterruptedException { boolean restarted; do { DistributedSystem ds = this.myDs; restarted = false; this.server.join(); if (this.stoppedForReconnect) { logger.info("waiting for distributed system to disconnect..."); while (ds.isConnected()) { Thread.sleep(5000); } logger.info("waiting for distributed system to reconnect..."); restarted = ds.waitUntilReconnected(-1, TimeUnit.SECONDS); if (restarted) { logger.info("system restarted"); } else { logger.info("system was not restarted"); } Thread restartThread = this.restartThread; if (restartThread != null) { logger.info("waiting for services to restart..."); restartThread.join(); this.restartThread = null; logger.info("done waiting for services to restart"); } } } while (restarted); } /** * launch a thread that will restart location services */ private void launchRestartThread() { // create a thread group having a last-chance exception-handler ThreadGroup group = LoggingThreadGroup.createThreadGroup("Locator restart thread group"); // TODO: non-atomic operation on volatile field restartThread this.restartThread = new Thread(group, "Location services restart thread") { @Override public void run() { boolean restarted = false; try { restarted = attemptReconnect(); logger.info("attemptReconnect returned {}", restarted); } catch (InterruptedException e) { logger.info("attempt to restart location services was interrupted", e); } catch (IOException e) { logger.info("attempt to restart location services terminated", e); } finally { if (!restarted) { stoppedForReconnect = false; } } InternalLocator.this.restartThread = null; } }; this.restartThread.setDaemon(true); this.restartThread.start(); } /** * reconnects the locator to a restarting DistributedSystem. If quorum checks are enabled this * will start peer location services before a distributed system is available if the quorum check * succeeds. It will then wait for the system to finish reconnecting before returning. If quorum * checks are not being done this merely waits for the distributed system to reconnect and then * starts location services. * * @return true if able to reconnect the locator to the new distributed system */ private boolean attemptReconnect() throws InterruptedException, IOException { boolean restarted = false; if (this.stoppedForReconnect) { logger.info("attempting to restart locator"); boolean tcpServerStarted = false; InternalDistributedSystem ds = this.myDs; long waitTime = ds.getConfig().getMaxWaitTimeForReconnect() / 2; QuorumChecker checker = null; while (ds.getReconnectedSystem() == null && !ds.isReconnectCancelled()) { if (checker == null) { checker = this.myDs.getQuorumChecker(); if (checker != null) { logger.info("The distributed system returned this quorum checker: {}", checker); } } if (checker != null && !tcpServerStarted) { boolean start = checker.checkForQuorum(3 * this.myDs.getConfig().getMemberTimeout()); if (start) { // start up peer location. server location is started after the DS finishes // reconnecting logger.info("starting peer location"); if (this.locatorListener != null) { this.locatorListener.clearLocatorInfo(); } this.stoppedForReconnect = false; this.myDs = null; this.myCache = null; restartWithoutDS(); tcpServerStarted = true; setLocator(this); } } ds.waitUntilReconnected(waitTime, TimeUnit.MILLISECONDS); } InternalDistributedSystem newSystem = (InternalDistributedSystem) ds.getReconnectedSystem(); if (newSystem != null) { if (!tcpServerStarted) { if (this.locatorListener != null) { this.locatorListener.clearLocatorInfo(); } this.stoppedForReconnect = false; } restartWithDS(newSystem, GemFireCacheImpl.getInstance()); setLocator(this); restarted = true; } } logger.info("restart thread exiting. Service was {}restarted", restarted ? "" : "not "); return restarted; } private void restartWithoutDS() throws IOException { synchronized (locatorLock) { if (locator != this && hasLocator()) { throw new IllegalStateException( "A locator can not be created because one already exists in this JVM."); } this.myDs = null; this.myCache = null; logger.info("Locator restart: initializing TcpServer peer location services"); this.server.restarting(null, null, null); if (this.productUseLog.isClosed()) { this.productUseLog.reopen(); } if (!this.server.isAlive()) { logger.info("Locator restart: starting TcpServer"); startTcpServer(); } } } private void restartWithDS(InternalDistributedSystem newSystem, InternalCache newCache) throws IOException { synchronized (locatorLock) { if (locator != this && hasLocator()) { throw new IllegalStateException( "A locator can not be created because one already exists in this JVM."); } this.myDs = newSystem; this.myCache = newCache; this.myDs.setDependentLocator(this); logger.info("Locator restart: initializing TcpServer"); if (isSharedConfigurationEnabled()) { this.sharedConfig = new ClusterConfigurationService(newCache); } this.server.restarting(newSystem, newCache, this.sharedConfig); if (this.productUseLog.isClosed()) { this.productUseLog.reopen(); } this.productUseLog.monitorUse(newSystem); this.isSharedConfigurationStarted = true; if (isSharedConfigurationEnabled()) { ExecutorService es = newCache.getDistributionManager().getThreadPool(); es.execute(new SharedConfigurationRunnable()); } if (!this.server.isAlive()) { logger.info("Locator restart: starting TcpServer"); startTcpServer(); } logger.info("Locator restart: initializing JMX manager"); startJmxManagerLocationService(newCache); endStartLocator(this.myDs); logger.info("Locator restart completed"); } } @Override public DistributedSystem getDistributedSystem() { return this.myDs; } @Override public boolean isPeerLocator() { return this.peerLocator; } @Override public boolean isServerLocator() { return this.serverLocator != null; } /** * Returns null if no server locator; otherwise returns the advisee that represents the server * locator. */ public ServerLocator getServerLocatorAdvisee() { return this.serverLocator; } /** * Return the port on which the locator is actually listening. If called before the locator has * actually started, this method will return null. * * @return the port the locator is listening on or null if it has not yet been started */ @Override public Integer getPort() { if (this.server != null) { return this.server.getPort(); } return null; } class FetchSharedConfigStatus implements Callable<SharedConfigurationStatusResponse> { static final int SLEEPTIME = 1000; static final byte MAX_RETRIES = 5; @Override public SharedConfigurationStatusResponse call() throws InterruptedException { final InternalLocator locator = InternalLocator.this; // TODO: this for-loop is probably not necessary as the if to break is always true for (int i = 0; i < MAX_RETRIES; i++) { if (locator.sharedConfig != null) { SharedConfigurationStatus status = locator.sharedConfig.getStatus(); if (status != SharedConfigurationStatus.STARTED || status != SharedConfigurationStatus.NOT_STARTED) { break; } } Thread.sleep(SLEEPTIME); } SharedConfigurationStatusResponse response; if (locator.sharedConfig != null) { response = locator.sharedConfig.createStatusResponse(); } else { response = new SharedConfigurationStatusResponse(); response.setStatus(SharedConfigurationStatus.UNDETERMINED); } return response; } } public SharedConfigurationStatusResponse getSharedConfigurationStatus() { ExecutorService es = this.myCache.getDistributionManager().getWaitingThreadPool(); Future<SharedConfigurationStatusResponse> statusFuture = es.submit(new FetchSharedConfigStatus()); SharedConfigurationStatusResponse response; try { response = statusFuture.get(5, TimeUnit.SECONDS); } catch (Exception e) { logger.info("Exception occurred while fetching the status {}", CliUtil.stackTraceAsString(e)); response = new SharedConfigurationStatusResponse(); response.setStatus(SharedConfigurationStatus.UNDETERMINED); } return response; } public static class PrimaryHandler implements TcpHandler { private volatile HashMap<Class, TcpHandler> handlerMapping = new HashMap<>(); private volatile HashSet<TcpHandler> allHandlers = new HashSet<>(); private TcpServer tcpServer; private final LocatorMembershipListener locatorListener; private final InternalLocator internalLocator; // GEODE-2253 test condition private boolean hasWaitedForHandlerInitialization = false; PrimaryHandler(InternalLocator locator, LocatorMembershipListener listener) { this.locatorListener = listener; this.internalLocator = locator; } // this method is synchronized to make sure that no new handlers are added while // initialization is taking place. @Override public synchronized void init(TcpServer tcpServer) { if (this.locatorListener != null) { // This is deferred until now as the initial requested port could have been 0 this.locatorListener.setPort(this.internalLocator.getPort()); } this.tcpServer = tcpServer; for (TcpHandler handler : this.allHandlers) { handler.init(tcpServer); } } @Override public void restarting(DistributedSystem ds, GemFireCache cache, ClusterConfigurationService sharedConfig) { if (ds != null) { for (TcpHandler handler : this.allHandlers) { handler.restarting(ds, cache, sharedConfig); } } } @Override public Object processRequest(Object request) throws IOException { long giveup = 0; while (giveup == 0 || System.currentTimeMillis() < giveup) { TcpHandler handler; if (request instanceof PeerLocatorRequest) { handler = this.handlerMapping.get(PeerLocatorRequest.class); } else { handler = this.handlerMapping.get(request.getClass()); } if (handler != null) { return handler.processRequest(request); } else { if (this.locatorListener != null) { return this.locatorListener.handleRequest(request); } else { // either there is a configuration problem or the locator is still starting up if (giveup == 0) { int locatorWaitTime = this.internalLocator.getConfig().getLocatorWaitTime(); if (locatorWaitTime <= 0) { // always retry some number of times locatorWaitTime = 30; } this.hasWaitedForHandlerInitialization = true; giveup = System.currentTimeMillis() + locatorWaitTime * 1000; try { Thread.sleep(1000); } catch (InterruptedException ignored) { // running in an executor - no need to set the interrupted flag on the thread return null; } } } } } // while logger.info( "Received a location request of class {} but the handler for this is " + "either not enabled or is not ready to process requests", request.getClass().getSimpleName()); return null; } /** * GEODE-2253 test condition - has this handler waited for a subordinate handler to be * installed? */ public boolean hasWaitedForHandlerInitialization() { return this.hasWaitedForHandlerInitialization; } @Override public void shutDown() { try { for (TcpHandler handler : this.allHandlers) { handler.shutDown(); } } finally { this.internalLocator.handleShutdown(); } } synchronized boolean isHandled(Class clazz) { return this.handlerMapping.containsKey(clazz); } public synchronized void addHandler(Class clazz, TcpHandler handler) { HashMap<Class, TcpHandler> tmpHandlerMapping = new HashMap<>(this.handlerMapping); HashSet<TcpHandler> tmpAllHandlers = new HashSet<>(this.allHandlers); tmpHandlerMapping.put(clazz, handler); if (tmpAllHandlers.add(handler) && this.tcpServer != null) { handler.init(this.tcpServer); } this.handlerMapping = tmpHandlerMapping; this.allHandlers = tmpAllHandlers; } @Override public void endRequest(Object request, long startTime) { TcpHandler handler = this.handlerMapping.get(request.getClass()); if (handler != null) { handler.endRequest(request, startTime); } } @Override public void endResponse(Object request, long startTime) { TcpHandler handler = this.handlerMapping.get(request.getClass()); if (handler != null) { handler.endResponse(request, startTime); } } } @Override public void onConnect(InternalDistributedSystem sys) { try { this.stats.hookupStats(sys, SocketCreator.getLocalHost().getCanonicalHostName() + '-' + this.server.getBindAddress()); ClientProtocolService clientProtocolService = this.server.getClientProtocolService(); if (clientProtocolService != null) { clientProtocolService.initializeStatistics("LocatorStats", sys); } } catch (UnknownHostException e) { logger.warn(e); } } /** * Returns collection of locator strings representing every locator instance hosted by this * member. * * @see #getLocators() */ public static Collection<String> getLocatorStrings() { Collection<String> locatorStrings; try { Collection<DistributionLocatorId> locatorIds = DistributionLocatorId .asDistributionLocatorIds(getLocators()); locatorStrings = DistributionLocatorId.asStrings(locatorIds); } catch (UnknownHostException ignored) { locatorStrings = null; } if (locatorStrings == null || locatorStrings.isEmpty()) { return null; } else { return locatorStrings; } } /** * A helper object so that the TcpServer can record its stats to the proper place. Stats are only * recorded if a distributed system is started. */ protected class DelayedPoolStatHelper implements PoolStatHelper { @Override public void startJob() { stats.incRequestInProgress(1); } @Override public void endJob() { stats.incRequestInProgress(-1); } } private void startSharedConfigurationService(InternalCache internalCache) { installSharedConfigHandler(); if (this.config.getEnableClusterConfiguration() && !this.isSharedConfigurationStarted) { if (!isDedicatedLocator()) { logger.info("Cluster configuration service not enabled as it is only supported " + "in dedicated locators"); return; } ExecutorService es = internalCache.getDistributionManager().getThreadPool(); es.execute(new SharedConfigurationRunnable()); } else { logger.info("Cluster configuration service is disabled"); } } public void startJmxManagerLocationService(InternalCache internalCache) { if (internalCache.getJmxManagerAdvisor() != null) { if (!this.handler.isHandled(JmxManagerLocatorRequest.class)) { this.handler.addHandler(JmxManagerLocatorRequest.class, new JmxManagerLocator(internalCache)); } } } /** * Creates and installs the handler {@link ConfigurationRequestHandler} */ private void installSharedConfigDistribution() { if (!this.handler.isHandled(ConfigurationRequest.class)) { this.handler.addHandler(ConfigurationRequest.class, new ConfigurationRequestHandler(this.sharedConfig)); logger.info("ConfigRequestHandler installed"); } } private void installSharedConfigHandler() { if (!this.handler.isHandled(SharedConfigurationStatusRequest.class)) { this.handler.addHandler(SharedConfigurationStatusRequest.class, new SharedConfigurationStatusRequestHandler()); logger.info("SharedConfigStatusRequestHandler installed"); } } public boolean hasHandlerForClass(Class messageClass) { return this.handler.isHandled(messageClass); } }