Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.nifi.cluster.manager.impl; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; import java.io.Serializable; import java.net.URI; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.ListIterator; import java.util.Map; import java.util.NavigableSet; import java.util.Queue; import java.util.Set; import java.util.Timer; import java.util.TimerTask; import java.util.TreeMap; import java.util.TreeSet; import java.util.UUID; import java.util.concurrent.CompletionService; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorCompletionService; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.regex.Pattern; import javax.net.ssl.SSLContext; import javax.ws.rs.HttpMethod; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.StreamingOutput; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.OutputKeys; import javax.xml.transform.Transformer; import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerFactory; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; import org.apache.commons.lang3.StringUtils; import org.apache.nifi.admin.service.AuditService; import org.apache.nifi.annotation.lifecycle.OnAdded; import org.apache.nifi.annotation.lifecycle.OnRemoved; import org.apache.nifi.cluster.BulletinsPayload; import org.apache.nifi.cluster.HeartbeatPayload; import org.apache.nifi.cluster.context.ClusterContext; import org.apache.nifi.cluster.context.ClusterContextImpl; import org.apache.nifi.cluster.event.Event; import org.apache.nifi.cluster.event.EventManager; import org.apache.nifi.cluster.firewall.ClusterNodeFirewall; import org.apache.nifi.cluster.flow.ClusterDataFlow; import org.apache.nifi.cluster.flow.DaoException; import org.apache.nifi.cluster.flow.DataFlowManagementService; import org.apache.nifi.cluster.flow.PersistedFlowState; import org.apache.nifi.cluster.manager.HttpClusterManager; import org.apache.nifi.cluster.manager.HttpRequestReplicator; import org.apache.nifi.cluster.manager.HttpResponseMapper; import org.apache.nifi.cluster.manager.NodeResponse; import org.apache.nifi.cluster.manager.exception.ConnectingNodeMutableRequestException; import org.apache.nifi.cluster.manager.exception.DisconnectedNodeMutableRequestException; import org.apache.nifi.cluster.manager.exception.IllegalClusterStateException; import org.apache.nifi.cluster.manager.exception.IllegalNodeDeletionException; import org.apache.nifi.cluster.manager.exception.IllegalNodeDisconnectionException; import org.apache.nifi.cluster.manager.exception.IllegalNodeReconnectionException; import org.apache.nifi.cluster.manager.exception.IneligiblePrimaryNodeException; import org.apache.nifi.cluster.manager.exception.NoConnectedNodesException; import org.apache.nifi.cluster.manager.exception.NoResponseFromNodesException; import org.apache.nifi.cluster.manager.exception.NodeDisconnectionException; import org.apache.nifi.cluster.manager.exception.NodeReconnectionException; import org.apache.nifi.cluster.manager.exception.PrimaryRoleAssignmentException; import org.apache.nifi.cluster.manager.exception.SafeModeMutableRequestException; import org.apache.nifi.cluster.manager.exception.UnknownNodeException; import org.apache.nifi.cluster.manager.exception.UriConstructionException; import org.apache.nifi.cluster.node.Node; import org.apache.nifi.cluster.node.Node.Status; import org.apache.nifi.cluster.protocol.ConnectionRequest; import org.apache.nifi.cluster.protocol.ConnectionResponse; import org.apache.nifi.cluster.protocol.Heartbeat; import org.apache.nifi.cluster.protocol.NodeBulletins; import org.apache.nifi.cluster.protocol.NodeIdentifier; import org.apache.nifi.cluster.protocol.ProtocolException; import org.apache.nifi.cluster.protocol.ProtocolHandler; import org.apache.nifi.cluster.protocol.StandardDataFlow; import org.apache.nifi.cluster.protocol.impl.ClusterManagerProtocolSenderListener; import org.apache.nifi.cluster.protocol.impl.ClusterServicesBroadcaster; import org.apache.nifi.cluster.protocol.message.ConnectionRequestMessage; import org.apache.nifi.cluster.protocol.message.ConnectionResponseMessage; import org.apache.nifi.cluster.protocol.message.ControllerStartupFailureMessage; import org.apache.nifi.cluster.protocol.message.DisconnectMessage; import org.apache.nifi.cluster.protocol.message.HeartbeatMessage; import org.apache.nifi.cluster.protocol.message.NodeBulletinsMessage; import org.apache.nifi.cluster.protocol.message.PrimaryRoleAssignmentMessage; import org.apache.nifi.cluster.protocol.message.ProtocolMessage; import org.apache.nifi.cluster.protocol.message.ProtocolMessage.MessageType; import org.apache.nifi.cluster.protocol.message.ReconnectionFailureMessage; import org.apache.nifi.cluster.protocol.message.ReconnectionRequestMessage; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.controller.ControllerService; import org.apache.nifi.controller.Heartbeater; import org.apache.nifi.controller.ReportingTaskNode; import org.apache.nifi.controller.ScheduledState; import org.apache.nifi.controller.StandardFlowSerializer; import org.apache.nifi.controller.StandardProcessorNode; import org.apache.nifi.controller.ValidationContextFactory; import org.apache.nifi.controller.exception.ComponentLifeCycleException; import org.apache.nifi.controller.queue.DropFlowFileState; import org.apache.nifi.controller.queue.ListFlowFileState; import org.apache.nifi.controller.reporting.ClusteredReportingTaskNode; import org.apache.nifi.controller.reporting.ReportingTaskInstantiationException; import org.apache.nifi.controller.reporting.ReportingTaskProvider; import org.apache.nifi.controller.reporting.StandardReportingInitializationContext; import org.apache.nifi.controller.scheduling.QuartzSchedulingAgent; import org.apache.nifi.controller.scheduling.StandardProcessScheduler; import org.apache.nifi.controller.scheduling.TimerDrivenSchedulingAgent; import org.apache.nifi.controller.service.ControllerServiceLoader; import org.apache.nifi.controller.service.ControllerServiceNode; import org.apache.nifi.controller.service.ControllerServiceProvider; import org.apache.nifi.controller.service.ControllerServiceState; import org.apache.nifi.controller.service.StandardControllerServiceProvider; import org.apache.nifi.controller.status.ProcessGroupStatus; import org.apache.nifi.controller.status.RemoteProcessGroupStatus; import org.apache.nifi.controller.status.history.ComponentStatusRepository; import org.apache.nifi.controller.status.history.MetricDescriptor; import org.apache.nifi.controller.status.history.StatusHistory; import org.apache.nifi.controller.status.history.StatusHistoryUtil; import org.apache.nifi.controller.status.history.StatusSnapshot; import org.apache.nifi.diagnostics.GarbageCollection; import org.apache.nifi.diagnostics.StorageUsage; import org.apache.nifi.diagnostics.SystemDiagnostics; import org.apache.nifi.encrypt.StringEncryptor; import org.apache.nifi.engine.FlowEngine; import org.apache.nifi.events.BulletinFactory; import org.apache.nifi.events.VolatileBulletinRepository; import org.apache.nifi.framework.security.util.SslContextFactory; import org.apache.nifi.io.socket.multicast.DiscoverableService; import org.apache.nifi.logging.ComponentLog; import org.apache.nifi.logging.ControllerServiceLogObserver; import org.apache.nifi.logging.LogLevel; import org.apache.nifi.logging.LogRepository; import org.apache.nifi.logging.LogRepositoryFactory; import org.apache.nifi.logging.NiFiLog; import org.apache.nifi.logging.ReportingTaskLogObserver; import org.apache.nifi.nar.ExtensionManager; import org.apache.nifi.nar.NarCloseable; import org.apache.nifi.nar.NarThreadContextClassLoader; import org.apache.nifi.processor.SimpleProcessLogger; import org.apache.nifi.processor.StandardValidationContextFactory; import org.apache.nifi.remote.RemoteResourceManager; import org.apache.nifi.remote.RemoteSiteListener; import org.apache.nifi.remote.SocketRemoteSiteListener; import org.apache.nifi.remote.cluster.ClusterNodeInformation; import org.apache.nifi.remote.cluster.NodeInformation; import org.apache.nifi.remote.protocol.socket.ClusterManagerServerProtocol; import org.apache.nifi.reporting.Bulletin; import org.apache.nifi.reporting.BulletinRepository; import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.reporting.ReportingInitializationContext; import org.apache.nifi.reporting.ReportingTask; import org.apache.nifi.reporting.Severity; import org.apache.nifi.scheduling.SchedulingStrategy; import org.apache.nifi.util.DomUtils; import org.apache.nifi.util.FormatUtils; import org.apache.nifi.util.NiFiProperties; import org.apache.nifi.util.ObjectHolder; import org.apache.nifi.util.ReflectionUtils; import org.apache.nifi.web.OptimisticLockingManager; import org.apache.nifi.web.Revision; import org.apache.nifi.web.UpdateRevision; import org.apache.nifi.web.api.dto.ControllerServiceDTO; import org.apache.nifi.web.api.dto.ControllerServiceReferencingComponentDTO; import org.apache.nifi.web.api.dto.DropRequestDTO; import org.apache.nifi.web.api.dto.FlowFileSummaryDTO; import org.apache.nifi.web.api.dto.FlowSnippetDTO; import org.apache.nifi.web.api.dto.ListingRequestDTO; import org.apache.nifi.web.api.dto.NodeDTO; import org.apache.nifi.web.api.dto.ProcessGroupDTO; import org.apache.nifi.web.api.dto.ProcessorDTO; import org.apache.nifi.web.api.dto.QueueSizeDTO; import org.apache.nifi.web.api.dto.RemoteProcessGroupContentsDTO; import org.apache.nifi.web.api.dto.RemoteProcessGroupDTO; import org.apache.nifi.web.api.dto.RemoteProcessGroupPortDTO; import org.apache.nifi.web.api.dto.ReportingTaskDTO; import org.apache.nifi.web.api.dto.provenance.ProvenanceDTO; import org.apache.nifi.web.api.dto.provenance.ProvenanceEventDTO; import org.apache.nifi.web.api.dto.provenance.ProvenanceRequestDTO; import org.apache.nifi.web.api.dto.provenance.ProvenanceResultsDTO; import org.apache.nifi.web.api.dto.status.ClusterStatusHistoryDTO; import org.apache.nifi.web.api.dto.status.NodeStatusHistoryDTO; import org.apache.nifi.web.api.dto.status.StatusHistoryDTO; import org.apache.nifi.web.api.dto.status.StatusSnapshotDTO; import org.apache.nifi.web.api.entity.ControllerServiceEntity; import org.apache.nifi.web.api.entity.ControllerServiceReferencingComponentsEntity; import org.apache.nifi.web.api.entity.ControllerServicesEntity; import org.apache.nifi.web.api.entity.DropRequestEntity; import org.apache.nifi.web.api.entity.FlowSnippetEntity; import org.apache.nifi.web.api.entity.ListingRequestEntity; import org.apache.nifi.web.api.entity.ProcessGroupEntity; import org.apache.nifi.web.api.entity.ProcessorEntity; import org.apache.nifi.web.api.entity.ProcessorsEntity; import org.apache.nifi.web.api.entity.ProvenanceEntity; import org.apache.nifi.web.api.entity.ProvenanceEventEntity; import org.apache.nifi.web.api.entity.RemoteProcessGroupEntity; import org.apache.nifi.web.api.entity.RemoteProcessGroupsEntity; import org.apache.nifi.web.api.entity.ReportingTaskEntity; import org.apache.nifi.web.api.entity.ReportingTasksEntity; import org.apache.nifi.web.util.WebUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.DOMException; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.NodeList; import org.xml.sax.SAXException; import org.xml.sax.SAXParseException; import com.sun.jersey.api.client.ClientResponse; /** * Provides a cluster manager implementation. The manager federates incoming HTTP client requests to the nodes' external API using the HTTP protocol. The manager also communicates with nodes using the * nodes' internal socket protocol. * * The manager's socket address may broadcasted using multicast if a MulticastServiceBroadcaster instance is set on this instance. The manager instance must be started after setting the broadcaster. * * The manager may be configured with an EventManager for recording noteworthy lifecycle events (e.g., first heartbeat received, node status change). * * The start() and stop() methods must be called to initialize and stop the instance. * */ public class WebClusterManager implements HttpClusterManager, ProtocolHandler, ControllerServiceProvider, ReportingTaskProvider { public static final String ROOT_GROUP_ID_ALIAS = "root"; public static final String BULLETIN_CATEGORY = "Clustering"; private static final Logger logger = new NiFiLog(LoggerFactory.getLogger(WebClusterManager.class)); private static final Logger heartbeatLogger = new NiFiLog( LoggerFactory.getLogger("org.apache.nifi.cluster.heartbeat")); /** * The HTTP header to store a cluster context. An example of what may be stored in the context is a node's auditable actions in response to a cluster request. The cluster context is serialized * using Java's serialization mechanism and hex encoded. */ public static final String CLUSTER_CONTEXT_HTTP_HEADER = "X-ClusterContext"; /** * HTTP Header that stores a unique ID for each request that is replicated to the nodes. This is used for logging purposes so that request information, such as timing, can be correlated between * the NCM and the nodes */ public static final String REQUEST_ID_HEADER = "X-RequestID"; /** * The HTTP header that the NCM specifies to ask a node if they are able to process a given request. The value is always 150-NodeContinue. The node will respond with 150 CONTINUE if it is able to * process the request, 417 EXPECTATION_FAILED otherwise. */ public static final String NCM_EXPECTS_HTTP_HEADER = "X-NcmExpects"; public static final int NODE_CONTINUE_STATUS_CODE = 150; /** * The HTTP header that the NCM specifies to indicate that a node should invalidate the specified user group. This is done to ensure that user cache is not stale when an administrator modifies a * group through the UI. */ public static final String CLUSTER_INVALIDATE_USER_GROUP_HEADER = "X-ClusterInvalidateUserGroup"; /** * The HTTP header that the NCM specifies to indicate that a node should invalidate the specified user. This is done to ensure that user cache is not stale when an administrator modifies a user * through the UI. */ public static final String CLUSTER_INVALIDATE_USER_HEADER = "X-ClusterInvalidateUser"; /** * The default number of seconds to respond to a connecting node if the manager cannot provide it with a current data flow. */ private static final int DEFAULT_CONNECTION_REQUEST_TRY_AGAIN_SECONDS = 5; public static final String DEFAULT_COMPONENT_STATUS_REPO_IMPLEMENTATION = "org.apache.nifi.controller.status.history.VolatileComponentStatusRepository"; public static final Pattern PROCESSORS_URI_PATTERN = Pattern .compile("/nifi-api/controller/process-groups/(?:(?:root)|(?:[a-f0-9\\-]{36}))/processors"); public static final Pattern PROCESSOR_URI_PATTERN = Pattern.compile( "/nifi-api/controller/process-groups/(?:(?:root)|(?:[a-f0-9\\-]{36}))/processors/[a-f0-9\\-]{36}"); public static final Pattern CLUSTER_PROCESSOR_URI_PATTERN = Pattern .compile("/nifi-api/cluster/processors/[a-f0-9\\-]{36}"); public static final Pattern REMOTE_PROCESS_GROUPS_URI_PATTERN = Pattern .compile("/nifi-api/controller/process-groups/(?:(?:root)|(?:[a-f0-9\\-]{36}))/remote-process-groups"); public static final Pattern REMOTE_PROCESS_GROUP_URI_PATTERN = Pattern.compile( "/nifi-api/controller/process-groups/(?:(?:root)|(?:[a-f0-9\\-]{36}))/remote-process-groups/[a-f0-9\\-]{36}"); public static final Pattern PROCESS_GROUP_URI_PATTERN = Pattern .compile("/nifi-api/controller/process-groups/(?:(?:root)|(?:[a-f0-9\\-]{36}))"); public static final Pattern TEMPLATE_INSTANCE_URI_PATTERN = Pattern .compile("/nifi-api/controller/process-groups/(?:(?:root)|(?:[a-f0-9\\-]{36}))/template-instance"); public static final Pattern FLOW_SNIPPET_INSTANCE_URI_PATTERN = Pattern .compile("/nifi-api/controller/process-groups/(?:(?:root)|(?:[a-f0-9\\-]{36}))/snippet-instance"); public static final String PROVENANCE_URI = "/nifi-api/controller/provenance"; public static final Pattern PROVENANCE_QUERY_URI = Pattern .compile("/nifi-api/controller/provenance/[a-f0-9\\-]{36}"); public static final Pattern PROVENANCE_EVENT_URI = Pattern .compile("/nifi-api/controller/provenance/events/[0-9]+"); public static final Pattern COUNTERS_URI = Pattern.compile("/nifi-api/controller/counters/[a-f0-9\\-]{36}"); public static final String CONTROLLER_SERVICES_URI = "/nifi-api/controller/controller-services/node"; public static final Pattern CONTROLLER_SERVICE_URI_PATTERN = Pattern .compile("/nifi-api/controller/controller-services/node/[a-f0-9\\-]{36}"); public static final Pattern CONTROLLER_SERVICE_REFERENCES_URI_PATTERN = Pattern .compile("/nifi-api/controller/controller-services/node/[a-f0-9\\-]{36}/references"); public static final String REPORTING_TASKS_URI = "/nifi-api/controller/reporting-tasks/node"; public static final Pattern REPORTING_TASK_URI_PATTERN = Pattern .compile("/nifi-api/controller/reporting-tasks/node/[a-f0-9\\-]{36}"); @Deprecated public static final Pattern QUEUE_CONTENTS_URI = Pattern.compile( "/nifi-api/controller/process-groups/(?:(?:root)|(?:[a-f0-9\\-]{36}))/connections/[a-f0-9\\-]{36}/contents"); public static final Pattern DROP_REQUESTS_URI = Pattern.compile( "/nifi-api/controller/process-groups/(?:(?:root)|(?:[a-f0-9\\-]{36}))/connections/[a-f0-9\\-]{36}/drop-requests"); public static final Pattern DROP_REQUEST_URI = Pattern.compile( "/nifi-api/controller/process-groups/(?:(?:root)|(?:[a-f0-9\\-]{36}))/connections/[a-f0-9\\-]{36}/drop-requests/[a-f0-9\\-]{36}"); public static final Pattern LISTING_REQUESTS_URI = Pattern.compile( "/nifi-api/controller/process-groups/(?:(?:root)|(?:[a-f0-9\\-]{36}))/connections/[a-f0-9\\-]{36}/listing-requests"); public static final Pattern LISTING_REQUEST_URI = Pattern.compile( "/nifi-api/controller/process-groups/(?:(?:root)|(?:[a-f0-9\\-]{36}))/connections/[a-f0-9\\-]{36}/listing-requests/[a-f0-9\\-]{36}"); private final NiFiProperties properties; private final HttpRequestReplicator httpRequestReplicator; private final HttpResponseMapper httpResponseMapper; private final DataFlowManagementService dataFlowManagementService; private final ClusterManagerProtocolSenderListener senderListener; private final OptimisticLockingManager optimisticLockingManager; private final StringEncryptor encryptor; private final Queue<Heartbeat> pendingHeartbeats = new ConcurrentLinkedQueue<>(); private final ReentrantReadWriteLock resourceRWLock = new ReentrantReadWriteLock(true); private final ClusterManagerLock readLock = new ClusterManagerLock(resourceRWLock.readLock(), "Read"); private final ClusterManagerLock writeLock = new ClusterManagerLock(resourceRWLock.writeLock(), "Write"); private final Set<Node> nodes = new HashSet<>(); private final ConcurrentMap<String, ReportingTaskNode> reportingTasks = new ConcurrentHashMap<>(); // null means the dataflow should be read from disk private StandardDataFlow cachedDataFlow = null; private NodeIdentifier primaryNodeId = null; private Timer heartbeatMonitor; private Timer heartbeatProcessor; private volatile ClusterServicesBroadcaster servicesBroadcaster = null; private volatile EventManager eventManager = null; private volatile ClusterNodeFirewall clusterFirewall = null; private volatile AuditService auditService = null; private volatile ControllerServiceProvider controllerServiceProvider = null; private final RemoteSiteListener remoteSiteListener; private final Integer remoteInputPort; private final Boolean remoteCommsSecure; private final BulletinRepository bulletinRepository; private final String instanceId; private final FlowEngine reportingTaskEngine; private final Map<NodeIdentifier, ComponentStatusRepository> componentMetricsRepositoryMap = new HashMap<>(); private final StandardProcessScheduler processScheduler; private final long componentStatusSnapshotMillis; public WebClusterManager(final HttpRequestReplicator httpRequestReplicator, final HttpResponseMapper httpResponseMapper, final DataFlowManagementService dataFlowManagementService, final ClusterManagerProtocolSenderListener senderListener, final NiFiProperties properties, final StringEncryptor encryptor, final OptimisticLockingManager optimisticLockingManager) { if (httpRequestReplicator == null) { throw new IllegalArgumentException("HttpRequestReplicator may not be null."); } else if (httpResponseMapper == null) { throw new IllegalArgumentException("HttpResponseMapper may not be null."); } else if (dataFlowManagementService == null) { throw new IllegalArgumentException("DataFlowManagementService may not be null."); } else if (senderListener == null) { throw new IllegalArgumentException("ClusterManagerProtocolSenderListener may not be null."); } else if (properties == null) { throw new IllegalArgumentException("NiFiProperties may not be null."); } // Ensure that our encryptor/decryptor is properly initialized this.httpRequestReplicator = httpRequestReplicator; this.httpResponseMapper = httpResponseMapper; this.dataFlowManagementService = dataFlowManagementService; this.properties = properties; this.bulletinRepository = new VolatileBulletinRepository(); this.instanceId = UUID.randomUUID().toString(); this.senderListener = senderListener; this.encryptor = encryptor; this.optimisticLockingManager = optimisticLockingManager; senderListener.addHandler(this); senderListener.setBulletinRepository(bulletinRepository); final String snapshotFrequency = properties.getProperty(NiFiProperties.COMPONENT_STATUS_SNAPSHOT_FREQUENCY, NiFiProperties.DEFAULT_COMPONENT_STATUS_SNAPSHOT_FREQUENCY); long snapshotMillis; try { snapshotMillis = FormatUtils.getTimeDuration(snapshotFrequency, TimeUnit.MILLISECONDS); } catch (final Exception e) { snapshotMillis = FormatUtils.getTimeDuration(NiFiProperties.DEFAULT_COMPONENT_STATUS_SNAPSHOT_FREQUENCY, TimeUnit.MILLISECONDS); } componentStatusSnapshotMillis = snapshotMillis; Executors.newSingleThreadScheduledExecutor().scheduleWithFixedDelay(new Runnable() { @Override public void run() { readLock.lock(); try { for (final Node node : nodes) { if (Status.CONNECTED.equals(node.getStatus())) { ComponentStatusRepository statusRepository = componentMetricsRepositoryMap .get(node.getNodeId()); if (statusRepository == null) { statusRepository = createComponentStatusRepository(); componentMetricsRepositoryMap.put(node.getNodeId(), statusRepository); } // ensure this node has a payload if (node.getHeartbeat() != null && node.getHeartbeatPayload() != null) { // if nothing has been captured or the current heartbeat is newer, capture it - comparing the heatbeat created timestamp // is safe since its marked as XmlTransient so we're assured that its based off the same clock that created the last capture date if (statusRepository.getLastCaptureDate() == null || node.getHeartbeat() .getCreatedTimestamp() > statusRepository.getLastCaptureDate().getTime()) { statusRepository.capture(node.getHeartbeatPayload().getProcessGroupStatus()); } } } } } catch (final Throwable t) { logger.warn("Unable to capture component metrics from Node heartbeats: " + t); if (logger.isDebugEnabled()) { logger.warn("", t); } } finally { readLock.unlock("capture component metrics from node heartbeats"); } } }, componentStatusSnapshotMillis, componentStatusSnapshotMillis, TimeUnit.MILLISECONDS); remoteInputPort = properties.getRemoteInputPort(); if (remoteInputPort == null) { remoteSiteListener = null; remoteCommsSecure = null; } else { // Register the ClusterManagerServerProtocol as the appropriate resource for site-to-site Server Protocol RemoteResourceManager.setServerProtocolImplementation(ClusterManagerServerProtocol.RESOURCE_NAME, ClusterManagerServerProtocol.class); remoteCommsSecure = properties.isSiteToSiteSecure(); if (remoteCommsSecure) { final SSLContext sslContext = SslContextFactory.createSslContext(properties, false); if (sslContext == null) { throw new IllegalStateException( "NiFi Configured to allow Secure Site-to-Site communications but the Keystore/Truststore properties are not configured"); } remoteSiteListener = new SocketRemoteSiteListener(remoteInputPort.intValue(), sslContext, this); } else { remoteSiteListener = new SocketRemoteSiteListener(remoteInputPort.intValue(), null, this); } } reportingTaskEngine = new FlowEngine(8, "Reporting Task Thread"); processScheduler = new StandardProcessScheduler(new Heartbeater() { @Override public void heartbeat() { } }, this, encryptor); // When we construct the scheduling agents, we can pass null for a lot of the arguments because we are only // going to be scheduling Reporting Tasks. Otherwise, it would not be okay. processScheduler.setSchedulingAgent(SchedulingStrategy.TIMER_DRIVEN, new TimerDrivenSchedulingAgent(null, reportingTaskEngine, null, encryptor)); processScheduler.setSchedulingAgent(SchedulingStrategy.CRON_DRIVEN, new QuartzSchedulingAgent(null, reportingTaskEngine, null, encryptor)); processScheduler.setMaxThreadCount(SchedulingStrategy.TIMER_DRIVEN, 10); processScheduler.setMaxThreadCount(SchedulingStrategy.CRON_DRIVEN, 10); controllerServiceProvider = new StandardControllerServiceProvider(processScheduler, bulletinRepository); } public void start() throws IOException { writeLock.lock(); try { if (isRunning()) { throw new IllegalStateException("Instance is already started."); } try { // setup heartbeat monitoring heartbeatMonitor = new Timer("Heartbeat Monitor", /* is daemon */ true); heartbeatMonitor.schedule(new HeartbeatMonitoringTimerTask(), 0, getHeartbeatMonitoringIntervalSeconds() * 1000); heartbeatProcessor = new Timer("Process Pending Heartbeats", true); final int processPendingHeartbeatDelay = 1000 * Math.max(1, getClusterProtocolHeartbeatSeconds() / 2); heartbeatProcessor.schedule(new ProcessPendingHeartbeatsTask(), processPendingHeartbeatDelay, processPendingHeartbeatDelay); // start request replication service httpRequestReplicator.start(); // start protocol service senderListener.start(); // start flow management service dataFlowManagementService.start(); if (remoteSiteListener != null) { remoteSiteListener.start(); } // load flow final ClusterDataFlow clusterDataFlow; if (dataFlowManagementService.isFlowCurrent()) { clusterDataFlow = dataFlowManagementService.loadDataFlow(); cachedDataFlow = clusterDataFlow.getDataFlow(); primaryNodeId = clusterDataFlow.getPrimaryNodeId(); } else { throw new IOException("Flow is not current."); } final byte[] serializedServices = clusterDataFlow.getControllerServices(); if (serializedServices != null && serializedServices.length > 0) { ControllerServiceLoader.loadControllerServices(this, new ByteArrayInputStream(serializedServices), encryptor, bulletinRepository, properties.getAutoResumeState()); } // start multicast broadcasting service, if configured if (servicesBroadcaster != null) { servicesBroadcaster.start(); } // start in safe mode executeSafeModeTask(); // Load and start running Reporting Tasks final byte[] serializedReportingTasks = clusterDataFlow.getReportingTasks(); if (serializedReportingTasks != null && serializedReportingTasks.length > 0) { loadReportingTasks(serializedReportingTasks); } } catch (final IOException ioe) { logger.warn("Failed to initialize cluster services due to: " + ioe, ioe); stop(); throw ioe; } } finally { writeLock.unlock("START"); } } public void stop() throws IOException { writeLock.lock(); try { // returns true if any service is running if (isRunning() == false) { throw new IllegalArgumentException("Instance is already stopped."); } boolean encounteredException = false; // stop the heartbeat monitoring if (isHeartbeatMonitorRunning()) { heartbeatMonitor.cancel(); heartbeatMonitor = null; } if (heartbeatProcessor != null) { heartbeatProcessor.cancel(); heartbeatProcessor = null; } // stop the HTTP request replicator service if (httpRequestReplicator.isRunning()) { httpRequestReplicator.stop(); } // stop the flow management service if (dataFlowManagementService.isRunning()) { dataFlowManagementService.stop(); } if (remoteSiteListener != null) { remoteSiteListener.stop(); } // stop the protocol listener service if (senderListener.isRunning()) { try { senderListener.stop(); } catch (final IOException ioe) { encounteredException = true; logger.warn("Failed to shutdown protocol service due to: " + ioe, ioe); } } // stop the service broadcaster if (isBroadcasting()) { servicesBroadcaster.stop(); } if (processScheduler != null) { processScheduler.shutdown(); } if (encounteredException) { throw new IOException( "Failed to shutdown Cluster Manager because one or more cluster services failed to shutdown. Check the logs for details."); } } finally { writeLock.unlock("STOP"); } } public boolean isRunning() { readLock.lock(); try { return isHeartbeatMonitorRunning() || httpRequestReplicator.isRunning() || senderListener.isRunning() || dataFlowManagementService.isRunning() || isBroadcasting(); } finally { readLock.unlock("isRunning"); } } @Override public boolean canHandle(ProtocolMessage msg) { return MessageType.CONNECTION_REQUEST == msg.getType() || MessageType.HEARTBEAT == msg.getType() || MessageType.CONTROLLER_STARTUP_FAILURE == msg.getType() || MessageType.BULLETINS == msg.getType() || MessageType.RECONNECTION_FAILURE == msg.getType(); } @Override public ProtocolMessage handle(final ProtocolMessage protocolMessage) throws ProtocolException { switch (protocolMessage.getType()) { case CONNECTION_REQUEST: return handleConnectionRequest((ConnectionRequestMessage) protocolMessage); case HEARTBEAT: final HeartbeatMessage heartbeatMessage = (HeartbeatMessage) protocolMessage; final Heartbeat original = heartbeatMessage.getHeartbeat(); final NodeIdentifier originalNodeId = original.getNodeIdentifier(); final Heartbeat heartbeatWithDn = new Heartbeat( addRequestorDn(originalNodeId, heartbeatMessage.getRequestorDN()), original.isPrimary(), original.isConnected(), original.getPayload()); handleHeartbeat(heartbeatWithDn); return null; case CONTROLLER_STARTUP_FAILURE: new Thread(new Runnable() { @Override public void run() { handleControllerStartupFailure((ControllerStartupFailureMessage) protocolMessage); } }, "Handle Controller Startup Failure Message from " + ((ControllerStartupFailureMessage) protocolMessage).getNodeId()).start(); return null; case RECONNECTION_FAILURE: new Thread(new Runnable() { @Override public void run() { handleReconnectionFailure((ReconnectionFailureMessage) protocolMessage); } }, "Handle Reconnection Failure Message from " + ((ReconnectionFailureMessage) protocolMessage).getNodeId()).start(); return null; case BULLETINS: final NodeBulletinsMessage bulletinsMessage = (NodeBulletinsMessage) protocolMessage; handleBulletins(bulletinsMessage.getBulletins()); return null; default: throw new ProtocolException("No handler defined for message type: " + protocolMessage.getType()); } } /** * Services connection requests. If the data flow management service is unable to provide a current copy of the data flow, then the returned connection response will indicate the node should try * later. Otherwise, the connection response will contain the the flow and the node identifier. * * If this instance is configured with a firewall and the request is blocked, then the response will not contain a node identifier. * * @param request a connection request * * @return a connection response */ @Override public ConnectionResponse requestConnection(final ConnectionRequest request) { final boolean lockObtained = writeLock.tryLock(3, TimeUnit.SECONDS); if (!lockObtained) { // Create try-later response because we are too busy to service the request right now. We do not want // to wait long because we want Node/NCM comms to be very responsive final int tryAgainSeconds; if (dataFlowManagementService.getRetrievalDelaySeconds() <= 0) { tryAgainSeconds = DEFAULT_CONNECTION_REQUEST_TRY_AGAIN_SECONDS; } else { tryAgainSeconds = dataFlowManagementService.getRetrievalDelaySeconds(); } // record event final String msg = "Connection requested from node, but manager was too busy to service request. Instructing node to try again in " + tryAgainSeconds + " seconds."; addEvent(request.getProposedNodeIdentifier(), msg); addBulletin(request.getProposedNodeIdentifier(), Severity.INFO, msg); // return try later response return new ConnectionResponse(tryAgainSeconds); } try { // resolve the proposed node identifier to a valid node identifier final NodeIdentifier resolvedNodeIdentifier = resolveProposedNodeIdentifier( request.getProposedNodeIdentifier()); if (isBlockedByFirewall(resolvedNodeIdentifier.getSocketAddress())) { // if the socket address is not listed in the firewall, then return a null response logger.info("Firewall blocked connection request from node " + resolvedNodeIdentifier); return ConnectionResponse.createBlockedByFirewallResponse(); } // get a raw reference to the node (if it doesn't exist, node will be null) Node node = getRawNode(resolvedNodeIdentifier.getId()); // create a new node if necessary and set status to connecting if (node == null) { node = new Node(resolvedNodeIdentifier, Status.CONNECTING); addEvent(node.getNodeId(), "Connection requested from new node. Setting status to connecting."); nodes.add(node); } else { node.setStatus(Status.CONNECTING); addEvent(resolvedNodeIdentifier, "Connection requested from existing node. Setting status to connecting"); } // record the time of the connection request node.setConnectionRequestedTimestamp(new Date().getTime()); // clear out old heartbeat info node.setHeartbeat(null); // try to obtain a current flow if (dataFlowManagementService.isFlowCurrent()) { // if a cached copy does not exist, load it from disk if (cachedDataFlow == null) { final ClusterDataFlow clusterDataFlow = dataFlowManagementService.loadDataFlow(); cachedDataFlow = clusterDataFlow.getDataFlow(); primaryNodeId = clusterDataFlow.getPrimaryNodeId(); } // determine if this node should be assigned the primary role final boolean primaryRole; if (primaryNodeId == null || primaryNodeId.logicallyEquals(node.getNodeId())) { setPrimaryNodeId(node.getNodeId()); addEvent(node.getNodeId(), "Setting primary role in connection response."); primaryRole = true; } else { primaryRole = false; } return new ConnectionResponse(node.getNodeId(), cachedDataFlow, primaryRole, remoteInputPort, remoteCommsSecure, instanceId); } /* * The manager does not have a current copy of the data flow, * so it will instruct the node to try connecting at a later * time. Meanwhile, the flow will be locked down from user * changes because the node is marked as connecting. */ /* * Create try-later response based on flow retrieval delay to give * the flow management service a chance to retrieve a curren flow */ final int tryAgainSeconds; if (dataFlowManagementService.getRetrievalDelaySeconds() <= 0) { tryAgainSeconds = DEFAULT_CONNECTION_REQUEST_TRY_AGAIN_SECONDS; } else { tryAgainSeconds = dataFlowManagementService.getRetrievalDelaySeconds(); } // record event addEvent(node.getNodeId(), "Connection requested from node, but manager was unable to obtain current flow. Instructing node to try again in " + tryAgainSeconds + " seconds."); // return try later response return new ConnectionResponse(tryAgainSeconds); } finally { writeLock.unlock("requestConnection"); } } /** * Services reconnection requests for a given node. If the node indicates reconnection failure, then the node will be set to disconnected and if the node has primary role, then the role will be * revoked. Otherwise, a reconnection request will be sent to the node, initiating the connection handshake. * * @param nodeId a node identifier * * @throws UnknownNodeException if the node does not exist * @throws IllegalNodeReconnectionException if the node cannot be reconnected because the node is not disconnected * @throws NodeReconnectionException if the reconnection message failed to be sent or the cluster could not provide a current data flow for the reconnection request */ @Override public void requestReconnection(final String nodeId, final String userDn) throws UnknownNodeException, IllegalNodeReconnectionException { Node node = null; final boolean primaryRole; final int tryAgainSeconds; writeLock.lock(); try { // check if we know about this node and that it is disconnected node = getRawNode(nodeId); logger.info("Request was made by {} to reconnect node {} to cluster", userDn, node == null ? nodeId : node); if (node == null) { throw new UnknownNodeException("Node does not exist."); } else if (Status.DISCONNECTED != node.getStatus()) { throw new IllegalNodeReconnectionException("Node must be disconnected before it can reconnect."); } // clear out old heartbeat info node.setHeartbeat(null); // get the dataflow to send with the reconnection request if (!dataFlowManagementService.isFlowCurrent()) { /* node remains disconnected */ final String msg = "Reconnection requested for node, but manager was unable to obtain current flow. Setting node to disconnected."; addEvent(node.getNodeId(), msg); addBulletin(node, Severity.WARNING, msg); throw new NodeReconnectionException( "Manager was unable to obtain current flow to provide in reconnection request to node. Try again in a few seconds."); } // if a cached copy does not exist, load it from disk if (cachedDataFlow == null) { final ClusterDataFlow clusterDataFlow = dataFlowManagementService.loadDataFlow(); cachedDataFlow = clusterDataFlow.getDataFlow(); primaryNodeId = clusterDataFlow.getPrimaryNodeId(); } node.setStatus(Status.CONNECTING); addEvent(node.getNodeId(), "Reconnection requested for node. Setting status to connecting."); // determine if this node should be assigned the primary role if (primaryNodeId == null || primaryNodeId.logicallyEquals(node.getNodeId())) { setPrimaryNodeId(node.getNodeId()); addEvent(node.getNodeId(), "Setting primary role in reconnection request."); primaryRole = true; } else { primaryRole = false; } if (dataFlowManagementService.getRetrievalDelaySeconds() <= 0) { tryAgainSeconds = DEFAULT_CONNECTION_REQUEST_TRY_AGAIN_SECONDS; } else { tryAgainSeconds = dataFlowManagementService.getRetrievalDelaySeconds(); } } catch (final UnknownNodeException | IllegalNodeReconnectionException | NodeReconnectionException une) { throw une; } catch (final Exception ex) { logger.warn("Problem encountered issuing reconnection request to node " + node.getNodeId() + " due to: " + ex, ex); node.setStatus(Status.DISCONNECTED); final String eventMsg = "Problem encountered issuing reconnection request. Node will remain disconnected: " + ex; addEvent(node.getNodeId(), eventMsg); addBulletin(node, Severity.WARNING, eventMsg); // Exception thrown will include node ID but event/bulletin do not because the node/id is passed along with the message throw new NodeReconnectionException("Problem encountered issuing reconnection request to " + node.getNodeId() + ". Node will remain disconnected: " + ex, ex); } finally { writeLock.unlock("requestReconnection"); } // Asynchronously start attempting reconnection. This is not completely thread-safe, as // we do this by releasing the write lock and then obtaining a read lock for each attempt, // so we suffer from the ABA problem. However, we are willing to accept the consequences of // this situation in order to avoid holding a lock for the entire duration. "The consequences" // are that a second thread could potentially be doing the same thing, issuing a reconnection request. // However, this is very unlikely to happen, based on the conditions under which we issue a reconnection // request. And if we do, the node will simply reconnect multiple times, which is not a big deal. requestReconnectionAsynchronously(node, primaryRole, 10, tryAgainSeconds); } private void requestReconnectionAsynchronously(final Node node, final boolean primaryRole, final int reconnectionAttempts, final int retrySeconds) { final Thread reconnectionThread = new Thread(new Runnable() { @Override public void run() { for (int i = 0; i < reconnectionAttempts; i++) { final ReconnectionRequestMessage request = new ReconnectionRequestMessage(); try { readLock.lock(); try { if (Status.CONNECTING != node.getStatus()) { // the node status has changed. It's no longer appropriate to attempt reconnection. return; } // create the request request.setNodeId(node.getNodeId()); request.setDataFlow(cachedDataFlow); request.setPrimary(primaryRole); request.setManagerRemoteSiteCommsSecure(remoteCommsSecure); request.setManagerRemoteSiteListeningPort(remoteInputPort); request.setInstanceId(instanceId); } finally { readLock.unlock("Reconnect " + node.getNodeId()); } // Issue a reconnection request to the node. senderListener.requestReconnection(request); node.setConnectionRequestedTimestamp(System.currentTimeMillis()); // successfully told node to reconnect -- we're done! return; } catch (final Exception e) { logger.warn("Problem encountered issuing reconnection request to node " + node.getNodeId() + " due to: " + e); if (logger.isDebugEnabled()) { logger.warn("", e); } addBulletin(node, Severity.WARNING, "Problem encountered issuing reconnection request to node " + node.getNodeId() + " due to: " + e); } try { Thread.sleep(1000L * retrySeconds); } catch (final InterruptedException ie) { break; } } // We failed to reconnect 10 times. We must now mark node as disconnected. writeLock.lock(); try { if (Status.CONNECTING == node.getStatus()) { requestDisconnectionQuietly(node.getNodeId(), "Failed to issue Reconnection Request " + reconnectionAttempts + " times"); } } finally { writeLock.unlock("Mark node as Disconnected as a result of reconnection failure"); } } }, "Reconnect " + node.getNodeId()); reconnectionThread.start(); } private Map<String, ReportingTaskNode> loadReportingTasks(final byte[] serialized) { final Map<String, ReportingTaskNode> tasks = new HashMap<>(); try { final Document document = parse(serialized); final NodeList tasksNodes = document.getElementsByTagName("reportingTasks"); final Element tasksElement = (Element) tasksNodes.item(0); //optional properties for all ReportingTasks for (final Element taskElement : DomUtils.getChildElementsByTagName(tasksElement, "reportingTask")) { //add global properties common to all tasks final Map<String, String> properties = new HashMap<>(); //get properties for the specific reporting task - id, name, class, //and schedulingPeriod must be set final String taskId = DomUtils.getChild(taskElement, "id").getTextContent().trim(); final String taskName = DomUtils.getChild(taskElement, "name").getTextContent().trim(); final List<Element> schedulingStrategyNodeList = DomUtils.getChildElementsByTagName(taskElement, "schedulingStrategy"); String schedulingStrategyValue = SchedulingStrategy.TIMER_DRIVEN.name(); if (schedulingStrategyNodeList.size() == 1) { final String specifiedValue = schedulingStrategyNodeList.get(0).getTextContent(); try { schedulingStrategyValue = SchedulingStrategy.valueOf(specifiedValue).name(); } catch (final Exception e) { throw new RuntimeException("Cannot start Reporting Task with id " + taskId + " because its Scheduling Strategy does not have a valid value", e); } } final SchedulingStrategy schedulingStrategy = SchedulingStrategy.valueOf(schedulingStrategyValue); final String taskSchedulingPeriod = DomUtils.getChild(taskElement, "schedulingPeriod") .getTextContent().trim(); final String taskClass = DomUtils.getChild(taskElement, "class").getTextContent().trim(); final String scheduleStateValue = DomUtils.getChild(taskElement, "scheduledState").getTextContent() .trim(); final ScheduledState scheduledState = ScheduledState.valueOf(scheduleStateValue); // Reporting Task Properties for (final Element property : DomUtils.getChildElementsByTagName(taskElement, "property")) { final String name = DomUtils.getChildText(property, "name"); final String value = DomUtils.getChildText(property, "value"); properties.put(name, value); } //set the class to be used for the configured reporting task final ReportingTaskNode reportingTaskNode; try { reportingTaskNode = createReportingTask(taskClass, taskId, false); } catch (final ReportingTaskInstantiationException e) { logger.error("Unable to load reporting task {} due to {}", new Object[] { taskId, e }); if (logger.isDebugEnabled()) { logger.error("", e); } continue; } final ReportingTask reportingTask = reportingTaskNode.getReportingTask(); final ComponentLog componentLog = new SimpleProcessLogger(taskId, reportingTask); final ReportingInitializationContext config = new StandardReportingInitializationContext(taskId, taskName, schedulingStrategy, taskSchedulingPeriod, componentLog, this); reportingTask.initialize(config); final String annotationData = DomUtils.getChildText(taskElement, "annotationData"); if (annotationData != null) { reportingTaskNode.setAnnotationData(annotationData.trim()); } final Map<PropertyDescriptor, String> resolvedProps; try (final NarCloseable narCloseable = NarCloseable.withNarLoader()) { resolvedProps = new HashMap<>(); for (final Map.Entry<String, String> entry : properties.entrySet()) { final PropertyDescriptor descriptor = reportingTask.getPropertyDescriptor(entry.getKey()); if (entry.getValue() == null) { resolvedProps.put(descriptor, descriptor.getDefaultValue()); } else { resolvedProps.put(descriptor, entry.getValue()); } } } for (final Map.Entry<PropertyDescriptor, String> entry : resolvedProps.entrySet()) { if (entry.getValue() != null) { reportingTaskNode.setProperty(entry.getKey().getName(), entry.getValue()); } } final String comments = DomUtils.getChildText(taskElement, "comment"); if (comments != null) { reportingTaskNode.setComments(comments); } reportingTaskNode.setScheduledState(scheduledState); if (ScheduledState.RUNNING.equals(scheduledState)) { if (reportingTaskNode.isValid()) { try { processScheduler.schedule(reportingTaskNode); } catch (final Exception e) { logger.error("Failed to start {} due to {}", reportingTaskNode, e); if (logger.isDebugEnabled()) { logger.error("", e); } } } else { logger.error("Failed to start {} because it is invalid due to {}", reportingTaskNode, reportingTaskNode.getValidationErrors()); } } tasks.put(reportingTaskNode.getIdentifier(), reportingTaskNode); } } catch (final SAXException | ParserConfigurationException | IOException | DOMException | NumberFormatException | InitializationException t) { logger.error("Unable to load reporting tasks due to {}", new Object[] { t }); if (logger.isDebugEnabled()) { logger.error("", t); } } return tasks; } @Override public ReportingTaskNode createReportingTask(final String type, final String id, final boolean firstTimeAdded) throws ReportingTaskInstantiationException { if (type == null) { throw new NullPointerException(); } ReportingTask task = null; final ClassLoader ctxClassLoader = Thread.currentThread().getContextClassLoader(); try { final ClassLoader detectedClassLoader = ExtensionManager.getClassLoader(type); final Class<?> rawClass; if (detectedClassLoader == null) { rawClass = Class.forName(type); } else { rawClass = Class.forName(type, false, detectedClassLoader); } Thread.currentThread().setContextClassLoader(detectedClassLoader); final Class<? extends ReportingTask> reportingTaskClass = rawClass.asSubclass(ReportingTask.class); final Object reportingTaskObj = reportingTaskClass.newInstance(); task = reportingTaskClass.cast(reportingTaskObj); } catch (final ClassNotFoundException | SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException t) { throw new ReportingTaskInstantiationException(type, t); } finally { if (ctxClassLoader != null) { Thread.currentThread().setContextClassLoader(ctxClassLoader); } } final ValidationContextFactory validationContextFactory = new StandardValidationContextFactory(this); final ReportingTaskNode taskNode = new ClusteredReportingTaskNode(task, id, processScheduler, new ClusteredEventAccess(this, auditService), bulletinRepository, controllerServiceProvider, validationContextFactory); taskNode.setName(task.getClass().getSimpleName()); reportingTasks.put(id, taskNode); if (firstTimeAdded) { try (final NarCloseable x = NarCloseable.withNarLoader()) { ReflectionUtils.invokeMethodsWithAnnotation(OnAdded.class, task); } catch (final Exception e) { throw new ComponentLifeCycleException("Failed to invoke On-Added Lifecycle methods of " + task, e); } } // Register log observer to provide bulletins when reporting task logs anything at WARN level or above final LogRepository logRepository = LogRepositoryFactory.getRepository(id); logRepository.addObserver(StandardProcessorNode.BULLETIN_OBSERVER_ID, LogLevel.WARN, new ReportingTaskLogObserver(getBulletinRepository(), taskNode)); return taskNode; } private Document parse(final byte[] serialized) throws SAXException, ParserConfigurationException, IOException { final DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance(); docFactory.setNamespaceAware(true); final DocumentBuilder builder = docFactory.newDocumentBuilder(); builder.setErrorHandler(new org.xml.sax.ErrorHandler() { @Override public void fatalError(final SAXParseException err) throws SAXException { logger.error("Config file line " + err.getLineNumber() + ", col " + err.getColumnNumber() + ", uri " + err.getSystemId() + " :message: " + err.getMessage()); if (logger.isDebugEnabled()) { logger.error("Error Stack Dump", err); } throw err; } @Override public void error(final SAXParseException err) throws SAXParseException { logger.error("Config file line " + err.getLineNumber() + ", col " + err.getColumnNumber() + ", uri " + err.getSystemId() + " :message: " + err.getMessage()); if (logger.isDebugEnabled()) { logger.error("Error Stack Dump", err); } throw err; } @Override public void warning(final SAXParseException err) throws SAXParseException { logger.warn(" Config file line " + err.getLineNumber() + ", uri " + err.getSystemId() + " : message : " + err.getMessage()); if (logger.isDebugEnabled()) { logger.warn("Warning stack dump", err); } throw err; } }); // build the docuemnt final Document document = builder.parse(new ByteArrayInputStream(serialized)); return document; } private void addBulletin(final Node node, final Severity severity, final String msg) { addBulletin(node.getNodeId(), severity, msg); } private void addBulletin(final NodeIdentifier nodeId, final Severity severity, final String msg) { bulletinRepository.addBulletin(BulletinFactory.createBulletin(BULLETIN_CATEGORY, severity.toString(), nodeId.getApiAddress() + ":" + nodeId.getApiPort() + " -- " + msg)); } /** * Services a disconnection request. * * @param nodeId a node identifier * @param userDn the DN of the user requesting the disconnection * * @throws UnknownNodeException if the node does not exist * @throws IllegalNodeDisconnectionException if the node cannot be disconnected due to the cluster's state (e.g., node is last connected node or node is primary) * @throws NodeDisconnectionException if the disconnection message fails to be sent. */ @Override public void requestDisconnection(final String nodeId, final String userDn) throws UnknownNodeException, IllegalNodeDisconnectionException, NodeDisconnectionException { writeLock.lock(); try { // check that the node is known final Node node = getNode(nodeId); if (node == null) { throw new UnknownNodeException("Node does not exist."); } requestDisconnection(node.getNodeId(), /* ignore last node */ false, "User " + userDn + " Disconnected Node"); } finally { writeLock.unlock("requestDisconnection(String)"); } } /** * Requests a disconnection to the node with the given node ID, but any exception thrown is suppressed. * * @param nodeId the node ID */ private void requestDisconnectionQuietly(final NodeIdentifier nodeId, final String explanation) { try { requestDisconnection(nodeId, /* ignore node check */ true, explanation); } catch (final IllegalNodeDisconnectionException | NodeDisconnectionException ex) { /* suppress exception */ } } /** * Issues a disconnection message to the node identified by the given node ID. If the node is not known, then a UnknownNodeException is thrown. If the node cannot be disconnected due to the * cluster's state and ignoreLastNodeCheck is false, then a IllegalNodeDisconnectionException is thrown. Otherwise, a disconnection message is issued to the node. * * Whether the disconnection message is successfully sent to the node, the node is marked as disconnected and if the node is the primary node, then the primary role is revoked. * * @param nodeId the ID of the node * @param ignoreNodeChecks if false, checks will be made to ensure the cluster supports the node's disconnection (e.g., the node is not the last connected node in the cluster; the node is not the * primary); otherwise, the request is made regardless of the cluster state * @param explanation * * @throws IllegalNodeDisconnectionException if the node cannot be disconnected due to the cluster's state (e.g., node is last connected node or node is primary). Not thrown if ignoreNodeChecks is * true. * @throws NodeDisconnectionException if the disconnection message fails to be sent. */ private void requestDisconnection(final NodeIdentifier nodeId, final boolean ignoreNodeChecks, final String explanation) throws IllegalNodeDisconnectionException, NodeDisconnectionException { writeLock.lock(); try { // check that the node is known final Node node = getRawNode(nodeId.getId()); if (node == null) { if (ignoreNodeChecks) { // issue the disconnection final DisconnectMessage request = new DisconnectMessage(); request.setNodeId(nodeId); request.setExplanation(explanation); addEvent(nodeId, "Disconnection requested due to " + explanation); senderListener.disconnect(request); addEvent(nodeId, "Node disconnected due to " + explanation); addBulletin(nodeId, Severity.INFO, "Node disconnected due to " + explanation); return; } else { throw new UnknownNodeException("Node does not exist"); } } // if necessary, check that the node may be disconnected if (!ignoreNodeChecks) { final Set<NodeIdentifier> connectedNodes = getNodeIds(Status.CONNECTED); // cannot disconnect the last connected node in the cluster if (connectedNodes.size() == 1 && connectedNodes.iterator().next().equals(nodeId)) { throw new IllegalNodeDisconnectionException( "Node may not be disconnected because it is the only connected node in the cluster."); } else if (isPrimaryNode(nodeId)) { // cannot disconnect the primary node in the cluster throw new IllegalNodeDisconnectionException( "Node may not be disconnected because it is the primary node in the cluster."); } } // update status node.setStatus(Status.DISCONNECTED); notifyDataFlowManagementServiceOfNodeStatusChange(); // issue the disconnection final DisconnectMessage request = new DisconnectMessage(); request.setNodeId(nodeId); request.setExplanation(explanation); addEvent(nodeId, "Disconnection requested due to " + explanation); senderListener.disconnect(request); addEvent(nodeId, "Node disconnected due to " + explanation); addBulletin(node, Severity.INFO, "Node disconnected due to " + explanation); } finally { writeLock.unlock("requestDisconnection(NodeIdentifier, boolean)"); } } /** * Messages the node to have the primary role. If the messaging fails, then the node is marked as disconnected. * * @param nodeId the node ID to assign primary role * * @return true if primary role assigned; false otherwise */ private boolean assignPrimaryRole(final NodeIdentifier nodeId) { writeLock.lock(); try { // create primary role message final PrimaryRoleAssignmentMessage msg = new PrimaryRoleAssignmentMessage(); msg.setNodeId(nodeId); msg.setPrimary(true); logger.info("Attempting to assign primary role to node: " + nodeId); // message senderListener.assignPrimaryRole(msg); logger.info("Assigned primary role to node: " + nodeId); addBulletin(nodeId, Severity.INFO, "Node assigned primary role"); // true indicates primary role assigned return true; } catch (final ProtocolException ex) { logger.warn("Failed attempt to assign primary role to node " + nodeId + " due to " + ex); addBulletin(nodeId, Severity.ERROR, "Failed to assign primary role to node due to: " + ex); // mark node as disconnected and log/record the event final Node node = getRawNode(nodeId.getId()); node.setStatus(Status.DISCONNECTED); addEvent(node.getNodeId(), "Disconnected because of failed attempt to assign primary role."); addBulletin(nodeId, Severity.WARNING, "Node disconnected because of failed attempt to assign primary role"); // false indicates primary role failed to be assigned return false; } finally { writeLock.unlock("assignPrimaryRole"); } } /** * Messages the node with the given node ID to no longer have the primary role. If the messaging fails, then the node is marked as disconnected. * * @return true if the primary role was revoked from the node; false otherwise */ private boolean revokePrimaryRole(final NodeIdentifier nodeId) { writeLock.lock(); try { // create primary role message final PrimaryRoleAssignmentMessage msg = new PrimaryRoleAssignmentMessage(); msg.setNodeId(nodeId); msg.setPrimary(false); logger.info("Attempting to revoke primary role from node: " + nodeId); // send message senderListener.assignPrimaryRole(msg); logger.info("Revoked primary role from node: " + nodeId); addBulletin(nodeId, Severity.INFO, "Primary Role revoked from node"); // true indicates primary role was revoked return true; } catch (final ProtocolException ex) { logger.warn("Failed attempt to revoke primary role from node " + nodeId + " due to " + ex); // mark node as disconnected and log/record the event final Node node = getRawNode(nodeId.getId()); node.setStatus(Status.DISCONNECTED); addEvent(node.getNodeId(), "Disconnected because of failed attempt to revoke primary role."); addBulletin(node, Severity.ERROR, "Node disconnected because of failed attempt to revoke primary role"); // false indicates primary role failed to be revoked return false; } finally { writeLock.unlock("revokePrimaryRole"); } } private NodeIdentifier addRequestorDn(final NodeIdentifier nodeId, final String dn) { return new NodeIdentifier(nodeId.getId(), nodeId.getApiAddress(), nodeId.getApiPort(), nodeId.getSocketAddress(), nodeId.getSocketPort(), dn); } private ConnectionResponseMessage handleConnectionRequest(final ConnectionRequestMessage requestMessage) { final NodeIdentifier proposedIdentifier = requestMessage.getConnectionRequest().getProposedNodeIdentifier(); final ConnectionRequest requestWithDn = new ConnectionRequest( addRequestorDn(proposedIdentifier, requestMessage.getRequestorDN())); final ConnectionResponse response = requestConnection(requestWithDn); final ConnectionResponseMessage responseMessage = new ConnectionResponseMessage(); responseMessage.setConnectionResponse(response); return responseMessage; } private void handleControllerStartupFailure(final ControllerStartupFailureMessage msg) { writeLock.lock(); try { final Node node = getRawNode(msg.getNodeId().getId()); if (node != null) { node.setStatus(Status.DISCONNECTED); addEvent(msg.getNodeId(), "Node could not join cluster because it failed to start up properly. Setting node to Disconnected. Node reported " + "the following error: " + msg.getExceptionMessage()); addBulletin(node, Severity.ERROR, "Node could not join cluster because it failed to start up properly. Setting node to Disconnected. Node " + "reported the following error: " + msg.getExceptionMessage()); } } finally { writeLock.unlock("handleControllerStartupFailure"); } } private void handleReconnectionFailure(final ReconnectionFailureMessage msg) { writeLock.lock(); try { final Node node = getRawNode(msg.getNodeId().getId()); if (node != null) { node.setStatus(Status.DISCONNECTED); final String errorMsg = "Node could not rejoin cluster. Setting node to Disconnected. Node reported the following error: " + msg.getExceptionMessage(); addEvent(msg.getNodeId(), errorMsg); addBulletin(node, Severity.ERROR, errorMsg); } } finally { writeLock.unlock("handleControllerStartupFailure"); } } @Override public ControllerServiceNode createControllerService(final String type, final String id, final boolean firstTimeAdded) { final ControllerServiceNode serviceNode = controllerServiceProvider.createControllerService(type, id, firstTimeAdded); // Register log observer to provide bulletins when reporting task logs anything at WARN level or above final LogRepository logRepository = LogRepositoryFactory.getRepository(id); logRepository.addObserver(StandardProcessorNode.BULLETIN_OBSERVER_ID, LogLevel.WARN, new ControllerServiceLogObserver(getBulletinRepository(), serviceNode)); return serviceNode; } @Override public ControllerService getControllerService(String serviceIdentifier) { return controllerServiceProvider.getControllerService(serviceIdentifier); } @Override public ControllerServiceNode getControllerServiceNode(final String id) { return controllerServiceProvider.getControllerServiceNode(id); } @Override public boolean isControllerServiceEnabled(final ControllerService service) { return controllerServiceProvider.isControllerServiceEnabled(service); } @Override public boolean isControllerServiceEnabled(final String serviceIdentifier) { return controllerServiceProvider.isControllerServiceEnabled(serviceIdentifier); } @Override public boolean isControllerServiceEnabling(final String serviceIdentifier) { return controllerServiceProvider.isControllerServiceEnabling(serviceIdentifier); } @Override public String getControllerServiceName(final String serviceIdentifier) { return controllerServiceProvider.getControllerServiceName(serviceIdentifier); } @Override public void removeControllerService(final ControllerServiceNode serviceNode) { controllerServiceProvider.removeControllerService(serviceNode); } @Override public void enableControllerService(final ControllerServiceNode serviceNode) { controllerServiceProvider.enableControllerService(serviceNode); } @Override public void enableControllerServices(final Collection<ControllerServiceNode> serviceNodes) { controllerServiceProvider.enableControllerServices(serviceNodes); } @Override public void disableControllerService(final ControllerServiceNode serviceNode) { controllerServiceProvider.disableControllerService(serviceNode); } @Override public Set<ControllerServiceNode> getAllControllerServices() { return controllerServiceProvider.getAllControllerServices(); } @Override public void disableReferencingServices(final ControllerServiceNode serviceNode) { controllerServiceProvider.disableReferencingServices(serviceNode); } @Override public void enableReferencingServices(final ControllerServiceNode serviceNode) { controllerServiceProvider.enableReferencingServices(serviceNode); } @Override public void scheduleReferencingComponents(final ControllerServiceNode serviceNode) { controllerServiceProvider.scheduleReferencingComponents(serviceNode); } @Override public void unscheduleReferencingComponents(final ControllerServiceNode serviceNode) { controllerServiceProvider.unscheduleReferencingComponents(serviceNode); } @Override public void verifyCanEnableReferencingServices(final ControllerServiceNode serviceNode) { controllerServiceProvider.verifyCanEnableReferencingServices(serviceNode); } @Override public void verifyCanScheduleReferencingComponents(final ControllerServiceNode serviceNode) { controllerServiceProvider.verifyCanScheduleReferencingComponents(serviceNode); } @Override public void verifyCanDisableReferencingServices(final ControllerServiceNode serviceNode) { controllerServiceProvider.verifyCanDisableReferencingServices(serviceNode); } @Override public void verifyCanStopReferencingComponents(final ControllerServiceNode serviceNode) { controllerServiceProvider.verifyCanStopReferencingComponents(serviceNode); } private byte[] serialize(final Document doc) throws TransformerException { final ByteArrayOutputStream baos = new ByteArrayOutputStream(); final DOMSource domSource = new DOMSource(doc); final StreamResult streamResult = new StreamResult(baos); // configure the transformer and convert the DOM final TransformerFactory transformFactory = TransformerFactory.newInstance(); final Transformer transformer = transformFactory.newTransformer(); transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2"); transformer.setOutputProperty(OutputKeys.INDENT, "yes"); // transform the document to byte stream transformer.transform(domSource, streamResult); return baos.toByteArray(); } private byte[] serializeControllerServices() throws ParserConfigurationException, TransformerException { final DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance(); docFactory.setNamespaceAware(true); final DocumentBuilder docBuilder = docFactory.newDocumentBuilder(); final Document document = docBuilder.newDocument(); final Element rootElement = document.createElement("controllerServices"); document.appendChild(rootElement); for (final ControllerServiceNode serviceNode : getAllControllerServices()) { StandardFlowSerializer.addControllerService(rootElement, serviceNode, encryptor); } return serialize(document); } private byte[] serializeReportingTasks() throws ParserConfigurationException, TransformerException { final DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance(); docFactory.setNamespaceAware(true); final DocumentBuilder docBuilder = docFactory.newDocumentBuilder(); final Document document = docBuilder.newDocument(); final Element rootElement = document.createElement("reportingTasks"); document.appendChild(rootElement); for (final ReportingTaskNode taskNode : getAllReportingTasks()) { StandardFlowSerializer.addReportingTask(rootElement, taskNode, encryptor); } return serialize(document); } public void saveControllerServices() { try { dataFlowManagementService.updateControllerServices(serializeControllerServices()); } catch (final Exception e) { logger.error( "Failed to save changes to NCM's Controller Services; changes may be lost on restart due to " + e); if (logger.isDebugEnabled()) { logger.error("", e); } getBulletinRepository().addBulletin(BulletinFactory.createBulletin("Controller Services", Severity.ERROR.name(), "Failed to save changes to NCM's Controller Services; changes may be lost on restart. See logs for more details.")); } } public void saveReportingTasks() { try { dataFlowManagementService.updateReportingTasks(serializeReportingTasks()); } catch (final Exception e) { logger.error( "Failed to save changes to NCM's Reporting Tasks; changes may be lost on restart due to " + e); if (logger.isDebugEnabled()) { logger.error("", e); } getBulletinRepository().addBulletin(BulletinFactory.createBulletin("Reporting Tasks", Severity.ERROR.name(), "Failed to save changes to NCM's Reporting Tasks; changes may be lost on restart. See logs for more details.")); } } @Override public Set<ReportingTaskNode> getAllReportingTasks() { readLock.lock(); try { return new HashSet<>(reportingTasks.values()); } finally { readLock.unlock("getReportingTasks"); } } @Override public ReportingTaskNode getReportingTaskNode(final String taskId) { readLock.lock(); try { return reportingTasks.get(taskId); } finally { readLock.unlock("getReportingTaskNode"); } } @Override public void startReportingTask(final ReportingTaskNode reportingTaskNode) { reportingTaskNode.verifyCanStart(); processScheduler.schedule(reportingTaskNode); } @Override public void stopReportingTask(final ReportingTaskNode reportingTaskNode) { reportingTaskNode.verifyCanStop(); processScheduler.unschedule(reportingTaskNode); } @Override public void removeReportingTask(final ReportingTaskNode reportingTaskNode) { writeLock.lock(); try { final ReportingTaskNode existing = reportingTasks.get(reportingTaskNode.getIdentifier()); if (existing == null || existing != reportingTaskNode) { throw new IllegalStateException( "Reporting Task " + reportingTaskNode + " does not exist in this Flow"); } reportingTaskNode.verifyCanDelete(); try (final NarCloseable x = NarCloseable.withNarLoader()) { ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnRemoved.class, reportingTaskNode.getReportingTask(), reportingTaskNode.getConfigurationContext()); } for (final Map.Entry<PropertyDescriptor, String> entry : reportingTaskNode.getProperties().entrySet()) { final PropertyDescriptor descriptor = entry.getKey(); if (descriptor.getControllerServiceDefinition() != null) { final String value = entry.getValue() == null ? descriptor.getDefaultValue() : entry.getValue(); if (value != null) { final ControllerServiceNode serviceNode = controllerServiceProvider .getControllerServiceNode(value); if (serviceNode != null) { serviceNode.removeReference(reportingTaskNode); } } } } reportingTasks.remove(reportingTaskNode.getIdentifier()); } finally { writeLock.unlock("removeReportingTask"); } } @Override public void disableReportingTask(final ReportingTaskNode reportingTask) { reportingTask.verifyCanDisable(); processScheduler.disableReportingTask(reportingTask); } @Override public void enableReportingTask(final ReportingTaskNode reportingTask) { reportingTask.verifyCanEnable(); processScheduler.enableReportingTask(reportingTask); } /** * Handle a bulletins message. * * @param bulletins bulletins */ public void handleBulletins(final NodeBulletins bulletins) { final NodeIdentifier nodeIdentifier = bulletins.getNodeIdentifier(); final String nodeAddress = nodeIdentifier.getApiAddress() + ":" + nodeIdentifier.getApiPort(); // unmarshal the message final BulletinsPayload payload = BulletinsPayload.unmarshal(bulletins.getPayload()); for (final Bulletin bulletin : payload.getBulletins()) { bulletin.setNodeAddress(nodeAddress); bulletinRepository.addBulletin(bulletin); } } /** * Handles a node's heartbeat. If this heartbeat is a node's first heartbeat since its connection request, then the manager will mark the node as connected. If the node was previously disconnected * due to a lack of heartbeat, then a reconnection request is issued. If the node was disconnected for other reasons, then a disconnection request is issued. If this instance is configured with a * firewall and the heartbeat is blocked, then a disconnection request is issued. */ @Override public void handleHeartbeat(final Heartbeat heartbeat) { // sanity check heartbeat if (heartbeat == null) { throw new IllegalArgumentException("Heartbeat may not be null."); } else if (heartbeat.getNodeIdentifier() == null) { throw new IllegalArgumentException("Heartbeat does not contain a node ID."); } /* * Processing a heartbeat requires a write lock, which may take a while * to obtain. Only the last heartbeat is necessary to process per node. * Futhermore, since many could pile up, heartbeats are processed in * bulk. * The below queue stores the pending heartbeats. */ pendingHeartbeats.add(heartbeat); } private void processPendingHeartbeats() { Node node; writeLock.lock(); try { /* * Get the most recent heartbeats for the nodes in the cluster. This * is achieved by "draining" the pending heartbeats queue, populating * a map that associates a node identifier with its latest heartbeat, and * finally, getting the values of the map. */ final Map<NodeIdentifier, Heartbeat> mostRecentHeartbeatsMap = new HashMap<>(); Heartbeat aHeartbeat; while ((aHeartbeat = pendingHeartbeats.poll()) != null) { mostRecentHeartbeatsMap.put(aHeartbeat.getNodeIdentifier(), aHeartbeat); } final Collection<Heartbeat> mostRecentHeartbeats = new ArrayList<>(mostRecentHeartbeatsMap.values()); // return fast if no work to do if (mostRecentHeartbeats.isEmpty()) { return; } logNodes("Before Heartbeat Processing", heartbeatLogger); final int numPendingHeartbeats = mostRecentHeartbeats.size(); if (heartbeatLogger.isDebugEnabled()) { heartbeatLogger.debug(String.format("Handling %s heartbeat%s", numPendingHeartbeats, numPendingHeartbeats > 1 ? "s" : "")); } for (final Heartbeat mostRecentHeartbeat : mostRecentHeartbeats) { try { // resolve the proposed node identifier to valid node identifier final NodeIdentifier resolvedNodeIdentifier = resolveProposedNodeIdentifier( mostRecentHeartbeat.getNodeIdentifier()); // get a raw reference to the node (if it doesn't exist, node will be null) node = getRawNode(resolvedNodeIdentifier.getId()); // if the node thinks it has the primary role, but the manager has assigned the role to a different node, then revoke the role if (mostRecentHeartbeat.isPrimary() && !isPrimaryNode(resolvedNodeIdentifier)) { addEvent(resolvedNodeIdentifier, "Heartbeat indicates node is running as primary node. Revoking primary role because primary role is assigned to a different node."); revokePrimaryRole(resolvedNodeIdentifier); } final boolean heartbeatIndicatesNotYetConnected = !mostRecentHeartbeat.isConnected(); if (isBlockedByFirewall(resolvedNodeIdentifier.getSocketAddress())) { if (node == null) { logger.info("Firewall blocked heartbeat received from unknown node " + resolvedNodeIdentifier + ". Issuing disconnection request."); } else { // record event addEvent(resolvedNodeIdentifier, "Firewall blocked received heartbeat. Issuing disconnection request."); } // request node to disconnect requestDisconnectionQuietly(resolvedNodeIdentifier, "Blocked By Firewall"); } else if (node == null) { // unknown node, so issue reconnect request // create new node and add to node set final Node newNode = new Node(resolvedNodeIdentifier, Status.DISCONNECTED); nodes.add(newNode); // record event addEvent(newNode.getNodeId(), "Received heartbeat from unknown node. Issuing reconnection request."); // record heartbeat newNode.setHeartbeat(mostRecentHeartbeat); requestReconnection(resolvedNodeIdentifier.getId(), "NCM Heartbeat Processing"); } else if (heartbeatIndicatesNotYetConnected) { if (Status.CONNECTED == node.getStatus()) { // record event addEvent(node.getNodeId(), "Received heartbeat from node that thinks it is not yet part of the cluster, though the Manager thought it " + "was. Marking as Disconnected and issuing reconnection request."); // record heartbeat node.setHeartbeat(null); node.setStatus(Status.DISCONNECTED); requestReconnection(resolvedNodeIdentifier.getId(), "NCM Heartbeat Processing"); } } else if (Status.DISCONNECTED == node.getStatus()) { // ignore heartbeats from nodes disconnected by means other than lack of heartbeat, unless it is // the only node. We allow it if it is the only node because if we have a one-node cluster, then // we cannot manually reconnect it. if (node.isHeartbeatDisconnection() || nodes.size() == 1) { // record event if (node.isHeartbeatDisconnection()) { addEvent(resolvedNodeIdentifier, "Received heartbeat from node previously disconnected due to lack of heartbeat. Issuing reconnection request."); } else { addEvent(resolvedNodeIdentifier, "Received heartbeat from node previously disconnected, but it is the only known node, so issuing reconnection request."); } // record heartbeat node.setHeartbeat(mostRecentHeartbeat); // request reconnection requestReconnection(resolvedNodeIdentifier.getId(), "NCM Heartbeat Processing"); } else { // disconnected nodes should not heartbeat, so we need to issue a disconnection request heartbeatLogger.info("Ignoring received heartbeat from disconnected node " + resolvedNodeIdentifier + ". Issuing disconnection request."); // request node to disconnect requestDisconnectionQuietly(resolvedNodeIdentifier, "Received Heartbeat from Node, but Manager has already marked Node as Disconnected"); } } else if (Status.DISCONNECTING == node.getStatus()) { /* ignore spurious heartbeat */ } else { // node is either either connected or connecting // first heartbeat causes status change from connecting to connected if (Status.CONNECTING == node.getStatus()) { if (mostRecentHeartbeat.getCreatedTimestamp() < node .getConnectionRequestedTimestamp()) { heartbeatLogger.info("Received heartbeat for node " + resolvedNodeIdentifier + " but ignoring because it was generated before the node was last asked to reconnect."); continue; } // set status to connected node.setStatus(Status.CONNECTED); // record event addEvent(resolvedNodeIdentifier, "Received first heartbeat from connecting node. Setting node to connected."); // notify service of updated node set notifyDataFlowManagementServiceOfNodeStatusChange(); addBulletin(node, Severity.INFO, "Node Connected"); } else { heartbeatLogger.info("Received heartbeat for node " + resolvedNodeIdentifier + "."); } // record heartbeat node.setHeartbeat(mostRecentHeartbeat); } } catch (final Exception e) { logger.error("Failed to process heartbeat from {}:{} due to {}", mostRecentHeartbeat.getNodeIdentifier().getApiAddress(), mostRecentHeartbeat.getNodeIdentifier().getApiPort(), e.toString()); if (logger.isDebugEnabled()) { logger.error("", e); } } } logNodes("After Heartbeat Processing", heartbeatLogger); } finally { writeLock.unlock("processPendingHeartbeats"); } } private ComponentStatusRepository createComponentStatusRepository() { final String implementationClassName = properties.getProperty( NiFiProperties.COMPONENT_STATUS_REPOSITORY_IMPLEMENTATION, DEFAULT_COMPONENT_STATUS_REPO_IMPLEMENTATION); if (implementationClassName == null) { throw new RuntimeException( "Cannot create Component Status Repository because the NiFi Properties is missing the following property: " + NiFiProperties.COMPONENT_STATUS_REPOSITORY_IMPLEMENTATION); } try { return NarThreadContextClassLoader.createInstance(implementationClassName, ComponentStatusRepository.class); } catch (final Exception e) { throw new RuntimeException(e); } } @Override public Set<Node> getNodes(final Status... statuses) { final Set<Status> desiredStatusSet = new HashSet<>(); for (final Status status : statuses) { desiredStatusSet.add(status); } readLock.lock(); try { final Set<Node> clonedNodes = new HashSet<>(); for (final Node node : nodes) { if (desiredStatusSet.isEmpty() || desiredStatusSet.contains(node.getStatus())) { clonedNodes.add(node.clone()); } } return Collections.unmodifiableSet(clonedNodes); } finally { readLock.unlock("getNodes(Status...)"); } } @Override public Node getNode(final String nodeId) { readLock.lock(); try { for (final Node node : nodes) { if (node.getNodeId().getId().equals(nodeId)) { return node.clone(); } } return null; } finally { readLock.unlock("getNode(String)"); } } @Override public Node getPrimaryNode() { readLock.lock(); try { if (primaryNodeId == null) { return null; } else { return getNode(primaryNodeId.getId()); } } finally { readLock.unlock("getPrimaryNode"); } } @Override public void deleteNode(final String nodeId, final String userDn) throws UnknownNodeException, IllegalNodeDeletionException { writeLock.lock(); try { final Node node = getNode(nodeId); if (node == null) { throw new UnknownNodeException("Node does not exist."); } else if (Status.DISCONNECTED == node.getStatus()) { nodes.remove(node); if (eventManager != null) { eventManager.clearEventHistory(node.getNodeId().getId()); } logger.info("Removing node {} from cluster because this action was requested by {}", node, userDn); } else { throw new IllegalNodeDeletionException("Node may not be deleted because it is not disconnected."); } } finally { writeLock.unlock("deleteNode"); } } @Override public Set<NodeIdentifier> getNodeIds(final Status... statuses) { readLock.lock(); try { final Set<NodeIdentifier> nodeIds = new HashSet<>(); for (final Node node : nodes) { if (statuses == null || statuses.length == 0) { nodeIds.add(node.getNodeId()); } else { for (final Node.Status status : statuses) { if (node.getStatus() == status) { nodeIds.add(node.getNodeId()); break; } } } } return nodeIds; } finally { readLock.unlock("getNodeIds(Status...)"); } } @Override public void setPrimaryNode(final String nodeId, final String userDn) throws UnknownNodeException, IneligiblePrimaryNodeException, PrimaryRoleAssignmentException { writeLock.lock(); try { final Node node = getNode(nodeId); if (node == null) { throw new UnknownNodeException("Node does not exist."); } else if (Status.CONNECTED != node.getStatus()) { throw new IneligiblePrimaryNodeException( "Node must be connected before it can be assigned as the primary node."); } // revoke primary role final Node primaryNode; if ((primaryNode = getPrimaryNode()) != null) { if (primaryNode.getStatus() == Status.DISCONNECTED) { throw new PrimaryRoleAssignmentException( "A disconnected, primary node exists. Delete the node before assigning the primary role to a different node."); } else if (revokePrimaryRole(primaryNode.getNodeId())) { addEvent(primaryNode.getNodeId(), "Role revoked from this node as part of primary role reassignment."); } else { throw new PrimaryRoleAssignmentException( "Failed to revoke primary role from node. Primary node is now disconnected. Delete the node before assigning the primary role to a different node."); } } // change the primary node ID to the given node setPrimaryNodeId(node.getNodeId()); // assign primary role if (assignPrimaryRole(node.getNodeId())) { addEvent(node.getNodeId(), "Role assigned to this node as part of primary role reassignment. Action performed by " + userDn); addBulletin(node, Severity.INFO, "Primary Role assigned to node by " + userDn); } else { throw new PrimaryRoleAssignmentException( "Cluster manager assigned primary role to node, but the node failed to accept the assignment. Cluster manager disconnected node."); } } finally { writeLock.unlock("setPrimaryNode"); } } private int getClusterProtocolHeartbeatSeconds() { return (int) FormatUtils.getTimeDuration(properties.getClusterProtocolHeartbeatInterval(), TimeUnit.SECONDS); } @Override public int getHeartbeatMonitoringIntervalSeconds() { return 4 * getClusterProtocolHeartbeatSeconds(); } @Override public int getMaxHeartbeatGapSeconds() { return 8 * getClusterProtocolHeartbeatSeconds(); } @Override public List<Event> getNodeEvents(final String nodeId) { readLock.lock(); try { List<Event> events = null; final EventManager eventMgr = eventManager; if (eventMgr != null) { events = eventMgr.getEvents(nodeId); } if (events == null) { return Collections.emptyList(); } else { return Collections.unmodifiableList(events); } } finally { readLock.unlock("getNodeEvents"); } } @Override public NodeResponse applyRequest(final String method, final URI uri, final Map<String, List<String>> parameters, final Map<String, String> headers) throws NoConnectedNodesException, NoResponseFromNodesException, UriConstructionException, ConnectingNodeMutableRequestException, DisconnectedNodeMutableRequestException, SafeModeMutableRequestException { return applyRequest(method, uri, parameters, headers, getNodeIds(Status.CONNECTED)); } @Override public NodeResponse applyRequest(final String method, final URI uri, final Map<String, List<String>> parameters, final Map<String, String> headers, final Set<NodeIdentifier> nodeIdentifiers) throws NoConnectedNodesException, NoResponseFromNodesException, UriConstructionException, ConnectingNodeMutableRequestException, DisconnectedNodeMutableRequestException, SafeModeMutableRequestException { final boolean mutableRequest = canChangeNodeState(method, uri); final ClusterManagerLock lock = mutableRequest ? writeLock : readLock; lock.lock(); try { // check that the request can be applied if (mutableRequest) { if (isInSafeMode()) { throw new SafeModeMutableRequestException( "Received a mutable request [" + method + " -- " + uri + "] while in safe mode"); } else if (!getNodeIds(Status.DISCONNECTED, Status.DISCONNECTING).isEmpty()) { throw new DisconnectedNodeMutableRequestException("Received a mutable request [" + method + " -- " + uri + "] while a node is disconnected from the cluster"); } else if (!getNodeIds(Status.CONNECTING).isEmpty()) { // if any node is connecting and a request can change the flow, then we throw an exception throw new ConnectingNodeMutableRequestException("Received a mutable request [" + method + " -- " + uri + "] while a node is trying to connect to the cluster"); } } final NodeResponse clientResponse = federateRequest(method, uri, parameters, null, headers, nodeIdentifiers); if (clientResponse == null) { if (mutableRequest) { throw new NoConnectedNodesException(String.format( "All nodes were disconnected as a result of applying request %s %s", method, uri)); } else { throw new NoResponseFromNodesException("No nodes were able to process this request."); } } else { return clientResponse; } } finally { lock.unlock( "applyRequest(String, URI, Map<String, List<String>>, Map<String, String>, Set<NodeIdentifier>"); } } @Override public NodeResponse applyRequest(final String method, final URI uri, final Object entity, final Map<String, String> headers) throws NoConnectedNodesException, NoResponseFromNodesException, UriConstructionException, ConnectingNodeMutableRequestException, DisconnectedNodeMutableRequestException, SafeModeMutableRequestException { return applyRequest(method, uri, entity, headers, getNodeIds(Status.CONNECTED)); } @Override public NodeResponse applyRequest(final String method, final URI uri, final Object entity, final Map<String, String> headers, final Set<NodeIdentifier> nodeIdentifiers) throws NoConnectedNodesException, NoResponseFromNodesException, UriConstructionException, ConnectingNodeMutableRequestException, DisconnectedNodeMutableRequestException, SafeModeMutableRequestException { final boolean mutableRequest = canChangeNodeState(method, uri); final ClusterManagerLock lock = mutableRequest ? writeLock : readLock; lock.lock(); try { // check that the request can be applied if (mutableRequest) { if (isInSafeMode()) { throw new SafeModeMutableRequestException( "Received a mutable request [" + method + " -- " + uri + "] while in safe mode"); } else if (!getNodeIds(Status.DISCONNECTED, Status.DISCONNECTING).isEmpty()) { throw new DisconnectedNodeMutableRequestException("Received a mutable request [" + method + " -- " + uri + "] while a node is disconnected from the cluster"); } else if (!getNodeIds(Status.CONNECTING).isEmpty()) { // if any node is connecting and a request can change the flow, then we throw an exception throw new ConnectingNodeMutableRequestException("Received a mutable request [" + method + " -- " + uri + "] while a node is trying to connect to the cluster"); } } final NodeResponse clientResponse = federateRequest(method, uri, null, entity, headers, nodeIdentifiers); if (clientResponse == null) { if (mutableRequest) { throw new NoConnectedNodesException(String.format( "All nodes were disconnected as a result of applying request %s %s", method, uri)); } else { throw new NoResponseFromNodesException("No nodes were able to process this request."); } } else { return clientResponse; } } finally { lock.unlock("applyRequest(String, URI, Object, Map<String, String>, Set<NodeIdentifier>"); } } public void setServicesBroadcaster(final ClusterServicesBroadcaster servicesBroadcaster) { writeLock.lock(); try { this.servicesBroadcaster = servicesBroadcaster; } finally { writeLock.unlock("setServicesBroadcaster"); } } public boolean addBroadcastedService(final DiscoverableService service) { writeLock.lock(); try { final ClusterServicesBroadcaster broadcaster = this.servicesBroadcaster; if (broadcaster == null) { throw new IllegalStateException("Service broadcasting is not configured."); } return broadcaster.addService(service); } finally { writeLock.unlock("addBroadcastedService"); } } public boolean removeBroadcastedService(final String serviceName) { writeLock.lock(); try { final ClusterServicesBroadcaster broadcaster = this.servicesBroadcaster; if (broadcaster == null) { throw new IllegalStateException("Service broadcasting is not configured."); } return broadcaster.removeService(serviceName); } finally { writeLock.unlock("removeBroadcastedService"); } } public boolean isBroadcastingConfigured() { readLock.lock(); try { return servicesBroadcaster != null; } finally { readLock.unlock("isBroadcastingConfigured"); } } public boolean isBroadcasting() { readLock.lock(); try { final ClusterServicesBroadcaster broadcaster = this.servicesBroadcaster; return broadcaster != null && broadcaster.isRunning(); } finally { readLock.unlock("isBroadcasting"); } } public void addEvent(final NodeIdentifier nodeId, String eventMsg) { writeLock.lock(); try { final Event event = new Event(nodeId.getId(), eventMsg); final EventManager eventMgr = eventManager; if (eventMgr != null) { eventMgr.addEvent(event); } logger.info(String.format("Node Event: %s -- '%s'", nodeId, eventMsg)); } finally { writeLock.unlock("addEvent"); } } public void setEventManager(final EventManager eventManager) { writeLock.lock(); try { this.eventManager = eventManager; } finally { writeLock.unlock("setEventManager"); } } public void setClusterFirewall(final ClusterNodeFirewall clusterFirewall) { writeLock.lock(); try { this.clusterFirewall = clusterFirewall; } finally { writeLock.unlock("setClusterFirewall"); } } public boolean isFirewallConfigured() { readLock.lock(); try { return clusterFirewall != null; } finally { readLock.unlock("isFirewallConfigured"); } } public void setAuditService(final AuditService auditService) { writeLock.lock(); try { this.auditService = auditService; } finally { writeLock.unlock("setAuditService"); } } public boolean isAuditingConfigured() { readLock.lock(); try { return auditService != null; } finally { readLock.unlock("isAuditingConfigured"); } } private boolean isPrimaryNode(final NodeIdentifier nodeId) { readLock.lock(); try { return primaryNodeId != null && primaryNodeId.equals(nodeId); } finally { readLock.unlock("isPrimaryNode"); } } private boolean isInSafeMode() { readLock.lock(); try { return primaryNodeId == null || getRawNode(primaryNodeId.getId()) == null; } finally { readLock.unlock("isInSafeMode"); } } private void setPrimaryNodeId(final NodeIdentifier primaryNodeId) throws DaoException { writeLock.lock(); try { dataFlowManagementService.updatePrimaryNode(primaryNodeId); // update the cached copy reference to minimize loading file from disk this.primaryNodeId = primaryNodeId; } finally { writeLock.unlock("setPrimaryNodeId"); } } // requires write lock to already be acquired unless method cannot change node state private NodeResponse federateRequest(final String method, final URI uri, final Map<String, List<String>> parameters, final Object entity, final Map<String, String> headers, final Set<NodeIdentifier> nodeIds) throws UriConstructionException { // ensure some nodes are connected if (nodeIds.isEmpty()) { throw new NoConnectedNodesException("Cannot apply " + method + " request to " + uri + " because there are currently no connected Nodes"); } logger.debug("Applying prototype request " + uri + " to nodes."); // the starting state of the flow (current, stale, unknown) final PersistedFlowState originalPersistedFlowState = dataFlowManagementService.getPersistedFlowState(); // check if this request can change the flow final boolean mutableRequest = canChangeNodeState(method, uri); final ObjectHolder<NodeResponse> holder = new ObjectHolder<>(null); final UpdateRevision federateRequest = new UpdateRevision() { @Override public Revision execute(Revision currentRevision) { // update headers to contain cluster contextual information to send to the node final Map<String, String> updatedHeaders = new HashMap<>(headers); final ClusterContext clusterCtx = new ClusterContextImpl(); clusterCtx.setRequestSentByClusterManager(true); // indicate request is sent from cluster manager clusterCtx.setRevision(currentRevision); // serialize cluster context and add to request header final String serializedClusterCtx = WebUtils.serializeObjectToHex(clusterCtx); updatedHeaders.put(CLUSTER_CONTEXT_HTTP_HEADER, serializedClusterCtx); // if the request is mutable, we need to verify that it is a valid request for all nodes in the cluster. if (mutableRequest) { updatedHeaders.put(NCM_EXPECTS_HTTP_HEADER, "150-NodeContinue"); final Set<NodeResponse> nodeResponses; if (entity == null) { nodeResponses = httpRequestReplicator.replicate(nodeIds, method, uri, parameters, updatedHeaders); } else { nodeResponses = httpRequestReplicator.replicate(nodeIds, method, uri, entity, updatedHeaders); } updatedHeaders.remove(NCM_EXPECTS_HTTP_HEADER); for (final NodeResponse response : nodeResponses) { if (response.getStatus() != NODE_CONTINUE_STATUS_CODE) { final String nodeDescription = response.getNodeId().getApiAddress() + ":" + response.getNodeId().getApiPort(); final ClientResponse clientResponse = response.getClientResponse(); if (clientResponse == null) { throw new IllegalClusterStateException("Node " + nodeDescription + " is unable to fulfill this request due to: Unexpected Response Code " + response.getStatus()); } final String nodeExplanation = clientResponse.getEntity(String.class); throw new IllegalClusterStateException("Node " + nodeDescription + " is unable to fulfill this request due to: " + nodeExplanation, response.getThrowable()); } } // set flow state to unknown to denote a mutable request replication in progress logger.debug("Setting Flow State to UNKNOWN due to mutable request to {} {}", method, uri); notifyDataFlowManagmentServiceOfFlowStateChange(PersistedFlowState.UNKNOWN); } // replicate request final Set<NodeResponse> nodeResponses; try { if (entity == null) { nodeResponses = httpRequestReplicator.replicate(nodeIds, method, uri, parameters, updatedHeaders); } else { nodeResponses = httpRequestReplicator.replicate(nodeIds, method, uri, entity, updatedHeaders); } } catch (final UriConstructionException uce) { // request was not replicated, so mark the flow with its original state if (mutableRequest) { notifyDataFlowManagmentServiceOfFlowStateChange(originalPersistedFlowState); } throw uce; } // merge the response final NodeResponse clientResponse = mergeResponses(uri, method, nodeResponses, mutableRequest); holder.set(clientResponse); // if we have a response get the updated cluster context for auditing and revision updating Revision updatedRevision = null; if (mutableRequest && clientResponse != null) { try { // get the cluster context from the response header final String serializedClusterContext = clientResponse.getClientResponse().getHeaders() .getFirst(CLUSTER_CONTEXT_HTTP_HEADER); if (StringUtils.isNotBlank(serializedClusterContext)) { // deserialize object final Serializable clusterContextObj = WebUtils .deserializeHexToObject(serializedClusterContext); // if we have a valid object, audit the actions if (clusterContextObj instanceof ClusterContext) { final ClusterContext clusterContext = (ClusterContext) clusterContextObj; if (auditService != null) { try { auditService.addActions(clusterContext.getActions()); } catch (final Throwable t) { logger.warn("Unable to record actions: " + t.getMessage()); if (logger.isDebugEnabled()) { logger.warn(StringUtils.EMPTY, t); } } } updatedRevision = clusterContext.getRevision(); } } } catch (final ClassNotFoundException cnfe) { logger.warn( "Classpath issue detected because failed to deserialize cluster context from node response due to: " + cnfe, cnfe); } } return updatedRevision; } }; // federate the request and lock on the revision if (mutableRequest) { optimisticLockingManager.setRevision(federateRequest); } else { federateRequest.execute(optimisticLockingManager.getLastModification().getRevision()); } return holder.get(); } private static boolean isProcessorsEndpoint(final URI uri, final String method) { return "GET".equalsIgnoreCase(method) && PROCESSORS_URI_PATTERN.matcher(uri.getPath()).matches(); } private static boolean isProcessorEndpoint(final URI uri, final String method) { if (("GET".equalsIgnoreCase(method) || "PUT".equalsIgnoreCase(method)) && (PROCESSOR_URI_PATTERN.matcher(uri.getPath()).matches() || CLUSTER_PROCESSOR_URI_PATTERN.matcher(uri.getPath()).matches())) { return true; } else if ("POST".equalsIgnoreCase(method) && PROCESSORS_URI_PATTERN.matcher(uri.getPath()).matches()) { return true; } return false; } private static boolean isProcessGroupEndpoint(final URI uri, final String method) { return ("GET".equalsIgnoreCase(method) || "PUT".equalsIgnoreCase(method)) && PROCESS_GROUP_URI_PATTERN.matcher(uri.getPath()).matches(); } private static boolean isTemplateEndpoint(final URI uri, final String method) { return "POST".equalsIgnoreCase(method) && TEMPLATE_INSTANCE_URI_PATTERN.matcher(uri.getPath()).matches(); } private static boolean isFlowSnippetEndpoint(final URI uri, final String method) { return "POST".equalsIgnoreCase(method) && FLOW_SNIPPET_INSTANCE_URI_PATTERN.matcher(uri.getPath()).matches(); } private static boolean isRemoteProcessGroupsEndpoint(final URI uri, final String method) { return "GET".equalsIgnoreCase(method) && REMOTE_PROCESS_GROUPS_URI_PATTERN.matcher(uri.getPath()).matches(); } private static boolean isRemoteProcessGroupEndpoint(final URI uri, final String method) { if (("GET".equalsIgnoreCase(method) || "PUT".equalsIgnoreCase(method)) && REMOTE_PROCESS_GROUP_URI_PATTERN.matcher(uri.getPath()).matches()) { return true; } else if ("POST".equalsIgnoreCase(method) && REMOTE_PROCESS_GROUPS_URI_PATTERN.matcher(uri.getPath()).matches()) { return true; } return false; } private static boolean isProvenanceQueryEndpoint(final URI uri, final String method) { if ("POST".equalsIgnoreCase(method) && PROVENANCE_URI.equals(uri.getPath())) { return true; } else if ("GET".equalsIgnoreCase(method) && PROVENANCE_QUERY_URI.matcher(uri.getPath()).matches()) { return true; } return false; } private static boolean isProvenanceEventEndpoint(final URI uri, final String method) { return "GET".equalsIgnoreCase(method) && PROVENANCE_EVENT_URI.matcher(uri.getPath()).matches(); } private static boolean isListFlowFilesEndpoint(final URI uri, final String method) { if (("GET".equalsIgnoreCase(method) || "DELETE".equalsIgnoreCase(method)) && LISTING_REQUEST_URI.matcher(uri.getPath()).matches()) { return true; } else if ("POST".equalsIgnoreCase(method) && LISTING_REQUESTS_URI.matcher(uri.getPath()).matches()) { return true; } return false; } private static boolean isCountersEndpoint(final URI uri) { return COUNTERS_URI.matcher(uri.getPath()).matches(); } private static boolean isControllerServicesEndpoint(final URI uri, final String method) { return "GET".equalsIgnoreCase(method) && CONTROLLER_SERVICES_URI.equals(uri.getPath()); } private static boolean isControllerServiceEndpoint(final URI uri, final String method) { if (("GET".equalsIgnoreCase(method) || "PUT".equalsIgnoreCase(method)) && CONTROLLER_SERVICE_URI_PATTERN.matcher(uri.getPath()).matches()) { return true; } else if ("POST".equalsIgnoreCase(method) && CONTROLLER_SERVICES_URI.equals(uri.getPath())) { return true; } return false; } private static boolean isControllerServiceReferenceEndpoint(final URI uri, final String method) { if (("GET".equalsIgnoreCase(method) || "PUT".equalsIgnoreCase(method)) && CONTROLLER_SERVICE_REFERENCES_URI_PATTERN.matcher(uri.getPath()).matches()) { return true; } return false; } private static boolean isReportingTasksEndpoint(final URI uri, final String method) { return "GET".equalsIgnoreCase(method) && REPORTING_TASKS_URI.equals(uri.getPath()); } private static boolean isReportingTaskEndpoint(final URI uri, final String method) { if (("GET".equalsIgnoreCase(method) || "PUT".equalsIgnoreCase(method)) && REPORTING_TASK_URI_PATTERN.matcher(uri.getPath()).matches()) { return true; } else if ("POST".equalsIgnoreCase(method) && REPORTING_TASKS_URI.equals(uri.getPath())) { return true; } return false; } private static boolean isDropRequestEndpoint(final URI uri, final String method) { if ("DELETE".equalsIgnoreCase(method) && QUEUE_CONTENTS_URI.matcher(uri.getPath()).matches()) { return true; } else if (("GET".equalsIgnoreCase(method) || "DELETE".equalsIgnoreCase(method)) && DROP_REQUEST_URI.matcher(uri.getPath()).matches()) { return true; } else if (("POST".equalsIgnoreCase(method) && DROP_REQUESTS_URI.matcher(uri.getPath()).matches())) { return true; } return false; } static boolean isResponseInterpreted(final URI uri, final String method) { return isProcessorsEndpoint(uri, method) || isProcessorEndpoint(uri, method) || isRemoteProcessGroupsEndpoint(uri, method) || isRemoteProcessGroupEndpoint(uri, method) || isProcessGroupEndpoint(uri, method) || isTemplateEndpoint(uri, method) || isFlowSnippetEndpoint(uri, method) || isProvenanceQueryEndpoint(uri, method) || isProvenanceEventEndpoint(uri, method) || isControllerServicesEndpoint(uri, method) || isControllerServiceEndpoint(uri, method) || isControllerServiceReferenceEndpoint(uri, method) || isReportingTasksEndpoint(uri, method) || isReportingTaskEndpoint(uri, method) || isDropRequestEndpoint(uri, method) || isListFlowFilesEndpoint(uri, method); } private void mergeProcessorValidationErrors(final ProcessorDTO processor, Map<NodeIdentifier, ProcessorDTO> processorMap) { final Map<String, Set<NodeIdentifier>> validationErrorMap = new HashMap<>(); for (final Map.Entry<NodeIdentifier, ProcessorDTO> nodeEntry : processorMap.entrySet()) { final NodeIdentifier nodeId = nodeEntry.getKey(); final ProcessorDTO nodeProcessor = nodeEntry.getValue(); // merge the validation errors mergeValidationErrors(validationErrorMap, nodeId, nodeProcessor.getValidationErrors()); } // set the merged the validation errors processor.setValidationErrors(normalizedMergedValidationErrors(validationErrorMap, processorMap.size())); } private void mergeProvenanceQueryResults(final ProvenanceDTO provenanceDto, final Map<NodeIdentifier, ProvenanceDTO> resultMap, final Set<NodeResponse> problematicResponses) { final ProvenanceResultsDTO results = provenanceDto.getResults(); final ProvenanceRequestDTO request = provenanceDto.getRequest(); final List<ProvenanceEventDTO> allResults = new ArrayList<>(1024); final Set<String> errors = new HashSet<>(); Date oldestEventDate = new Date(); int percentageComplete = 0; boolean finished = true; long totalRecords = 0; for (final Map.Entry<NodeIdentifier, ProvenanceDTO> entry : resultMap.entrySet()) { final NodeIdentifier nodeIdentifier = entry.getKey(); final String nodeAddress = nodeIdentifier.getApiAddress() + ":" + nodeIdentifier.getApiPort(); final ProvenanceDTO nodeDto = entry.getValue(); final ProvenanceResultsDTO nodeResultDto = nodeDto.getResults(); if (nodeResultDto != null && nodeResultDto.getProvenanceEvents() != null) { // increment the total number of records totalRecords += nodeResultDto.getTotalCount(); // populate the cluster identifier for (final ProvenanceEventDTO eventDto : nodeResultDto.getProvenanceEvents()) { eventDto.setClusterNodeId(nodeIdentifier.getId()); eventDto.setClusterNodeAddress(nodeAddress); // add node identifier to the event's id so that it is unique across cluster eventDto.setId(nodeIdentifier.getId() + eventDto.getId()); allResults.add(eventDto); } } if (nodeResultDto.getOldestEvent() != null && nodeResultDto.getOldestEvent().before(oldestEventDate)) { oldestEventDate = nodeResultDto.getOldestEvent(); } if (nodeResultDto.getErrors() != null) { for (final String error : nodeResultDto.getErrors()) { errors.add(nodeAddress + " -- " + error); } } percentageComplete += nodeDto.getPercentCompleted(); if (!nodeDto.isFinished()) { finished = false; } } percentageComplete /= resultMap.size(); // consider any problematic responses as errors for (final NodeResponse problematicResponse : problematicResponses) { final NodeIdentifier problemNode = problematicResponse.getNodeId(); final String problemNodeAddress = problemNode.getApiAddress() + ":" + problemNode.getApiPort(); errors.add(String.format("%s -- Request did not complete successfully (Status code: %s)", problemNodeAddress, problematicResponse.getStatus())); } // Since we get back up to the maximum number of results from each node, we need to sort those values and then // grab only the first X number of them. We do a sort based on time, such that the newest are included. // If 2 events have the same timestamp, we do a secondary sort based on Cluster Node Identifier. If those are // equal, we perform a terciary sort based on the the event id Collections.sort(allResults, new Comparator<ProvenanceEventDTO>() { @Override public int compare(final ProvenanceEventDTO o1, final ProvenanceEventDTO o2) { final int eventTimeComparison = o1.getEventTime().compareTo(o2.getEventTime()); if (eventTimeComparison != 0) { return -eventTimeComparison; } final String nodeId1 = o1.getClusterNodeId(); final String nodeId2 = o2.getClusterNodeId(); final int nodeIdComparison; if (nodeId1 == null && nodeId2 == null) { nodeIdComparison = 0; } else if (nodeId1 == null) { nodeIdComparison = 1; } else if (nodeId2 == null) { nodeIdComparison = -1; } else { nodeIdComparison = -nodeId1.compareTo(nodeId2); } if (nodeIdComparison != 0) { return nodeIdComparison; } return -Long.compare(o1.getEventId(), o2.getEventId()); } }); final int maxResults = request.getMaxResults().intValue(); final List<ProvenanceEventDTO> selectedResults; if (allResults.size() < maxResults) { selectedResults = allResults; } else { selectedResults = allResults.subList(0, maxResults); } // include any errors if (errors.size() > 0) { results.setErrors(errors); } results.setTotalCount(totalRecords); results.setTotal(FormatUtils.formatCount(totalRecords)); results.setProvenanceEvents(selectedResults); results.setOldestEvent(oldestEventDate); results.setGenerated(new Date()); provenanceDto.setPercentCompleted(percentageComplete); provenanceDto.setFinished(finished); } private void mergeRemoteProcessGroup(final RemoteProcessGroupDTO remoteProcessGroup, final Map<NodeIdentifier, RemoteProcessGroupDTO> remoteProcessGroupMap) { final RemoteProcessGroupContentsDTO remoteProcessGroupContents = remoteProcessGroup.getContents(); Boolean mergedIsTargetSecure = null; final List<String> mergedAuthorizationIssues = new ArrayList<>(); final Set<RemoteProcessGroupPortDTO> mergedInputPorts = new HashSet<>(); final Set<RemoteProcessGroupPortDTO> mergedOutputPorts = new HashSet<>(); for (final Map.Entry<NodeIdentifier, RemoteProcessGroupDTO> nodeEntry : remoteProcessGroupMap.entrySet()) { final NodeIdentifier nodeId = nodeEntry.getKey(); final RemoteProcessGroupDTO nodeRemoteProcessGroupDto = nodeEntry.getValue(); // merge the issues final List<String> nodeAuthorizationIssues = nodeRemoteProcessGroupDto.getAuthorizationIssues(); if (nodeAuthorizationIssues != null && !nodeAuthorizationIssues.isEmpty()) { for (final String nodeAuthorizationIssue : nodeAuthorizationIssues) { mergedAuthorizationIssues.add( nodeId.getApiAddress() + ":" + nodeId.getApiPort() + " -- " + nodeAuthorizationIssue); } } // use the first target secure flag since they will all be the same final Boolean nodeIsTargetSecure = nodeRemoteProcessGroupDto.isTargetSecure(); if (mergedIsTargetSecure == null) { mergedIsTargetSecure = nodeIsTargetSecure; } // merge the ports in the contents final RemoteProcessGroupContentsDTO nodeRemoteProcessGroupContentsDto = nodeRemoteProcessGroupDto .getContents(); if (remoteProcessGroupContents != null && nodeRemoteProcessGroupContentsDto != null) { if (nodeRemoteProcessGroupContentsDto.getInputPorts() != null) { mergedInputPorts.addAll(nodeRemoteProcessGroupContentsDto.getInputPorts()); } if (nodeRemoteProcessGroupContentsDto.getOutputPorts() != null) { mergedOutputPorts.addAll(nodeRemoteProcessGroupContentsDto.getOutputPorts()); } } } if (remoteProcessGroupContents != null) { if (!mergedInputPorts.isEmpty()) { remoteProcessGroupContents.setInputPorts(mergedInputPorts); } if (!mergedOutputPorts.isEmpty()) { remoteProcessGroupContents.setOutputPorts(mergedOutputPorts); } } if (mergedIsTargetSecure != null) { remoteProcessGroup.setTargetSecure(mergedIsTargetSecure); } if (!mergedAuthorizationIssues.isEmpty()) { remoteProcessGroup.setAuthorizationIssues(mergedAuthorizationIssues); } } private void mergeControllerServiceReferences( final Set<ControllerServiceReferencingComponentDTO> referencingComponents, final Map<NodeIdentifier, Set<ControllerServiceReferencingComponentDTO>> referencingComponentMap) { final Map<String, Integer> activeThreadCounts = new HashMap<>(); final Map<String, String> states = new HashMap<>(); for (final Map.Entry<NodeIdentifier, Set<ControllerServiceReferencingComponentDTO>> nodeEntry : referencingComponentMap .entrySet()) { final Set<ControllerServiceReferencingComponentDTO> nodeReferencingComponents = nodeEntry.getValue(); // go through all the nodes referencing components if (nodeReferencingComponents != null) { for (final ControllerServiceReferencingComponentDTO nodeReferencingComponent : nodeReferencingComponents) { // handle active thread counts if (nodeReferencingComponent.getActiveThreadCount() != null && nodeReferencingComponent.getActiveThreadCount() > 0) { final Integer current = activeThreadCounts.get(nodeReferencingComponent.getId()); if (current == null) { activeThreadCounts.put(nodeReferencingComponent.getId(), nodeReferencingComponent.getActiveThreadCount()); } else { activeThreadCounts.put(nodeReferencingComponent.getId(), nodeReferencingComponent.getActiveThreadCount() + current); } } // handle controller service state final String state = states.get(nodeReferencingComponent.getId()); if (state == null) { if (ControllerServiceState.DISABLING.name().equals(nodeReferencingComponent.getState())) { states.put(nodeReferencingComponent.getId(), ControllerServiceState.DISABLING.name()); } else if (ControllerServiceState.ENABLING.name() .equals(nodeReferencingComponent.getState())) { states.put(nodeReferencingComponent.getId(), ControllerServiceState.ENABLING.name()); } } } } } // go through each referencing components for (final ControllerServiceReferencingComponentDTO referencingComponent : referencingComponents) { final Integer activeThreadCount = activeThreadCounts.get(referencingComponent.getId()); if (activeThreadCount != null) { referencingComponent.setActiveThreadCount(activeThreadCount); } final String state = states.get(referencingComponent.getId()); if (state != null) { referencingComponent.setState(state); } } } private void mergeControllerService(final ControllerServiceDTO controllerService, final Map<NodeIdentifier, ControllerServiceDTO> controllerServiceMap) { final Map<String, Set<NodeIdentifier>> validationErrorMap = new HashMap<>(); final Set<ControllerServiceReferencingComponentDTO> referencingComponents = controllerService .getReferencingComponents(); final Map<NodeIdentifier, Set<ControllerServiceReferencingComponentDTO>> nodeReferencingComponentsMap = new HashMap<>(); String state = null; for (final Map.Entry<NodeIdentifier, ControllerServiceDTO> nodeEntry : controllerServiceMap.entrySet()) { final NodeIdentifier nodeId = nodeEntry.getKey(); final ControllerServiceDTO nodeControllerService = nodeEntry.getValue(); if (state == null) { if (ControllerServiceState.DISABLING.name().equals(nodeControllerService.getState())) { state = ControllerServiceState.DISABLING.name(); } else if (ControllerServiceState.ENABLING.name().equals(nodeControllerService.getState())) { state = ControllerServiceState.ENABLING.name(); } } for (final ControllerServiceReferencingComponentDTO nodeReferencingComponents : nodeControllerService .getReferencingComponents()) { nodeReferencingComponentsMap.put(nodeId, nodeReferencingComponents.getReferencingComponents()); } // merge the validation errors mergeValidationErrors(validationErrorMap, nodeId, nodeControllerService.getValidationErrors()); } // merge the referencing components mergeControllerServiceReferences(referencingComponents, nodeReferencingComponentsMap); // store the 'transition' state is applicable if (state != null) { controllerService.setState(state); } // set the merged the validation errors controllerService.setValidationErrors( normalizedMergedValidationErrors(validationErrorMap, controllerServiceMap.size())); } private void mergeReportingTask(final ReportingTaskDTO reportingTask, final Map<NodeIdentifier, ReportingTaskDTO> reportingTaskMap) { final Map<String, Set<NodeIdentifier>> validationErrorMap = new HashMap<>(); int activeThreadCount = 0; for (final Map.Entry<NodeIdentifier, ReportingTaskDTO> nodeEntry : reportingTaskMap.entrySet()) { final NodeIdentifier nodeId = nodeEntry.getKey(); final ReportingTaskDTO nodeReportingTask = nodeEntry.getValue(); if (nodeReportingTask.getActiveThreadCount() != null) { activeThreadCount += nodeReportingTask.getActiveThreadCount(); } // merge the validation errors mergeValidationErrors(validationErrorMap, nodeId, nodeReportingTask.getValidationErrors()); } // set the merged active thread counts reportingTask.setActiveThreadCount(activeThreadCount); // set the merged the validation errors reportingTask .setValidationErrors(normalizedMergedValidationErrors(validationErrorMap, reportingTaskMap.size())); } /** * Merges the validation errors into the specified map, recording the corresponding node identifier. * * @param validationErrorMap map * @param nodeId id * @param nodeValidationErrors errors */ public void mergeValidationErrors(final Map<String, Set<NodeIdentifier>> validationErrorMap, final NodeIdentifier nodeId, final Collection<String> nodeValidationErrors) { if (nodeValidationErrors != null) { for (final String nodeValidationError : nodeValidationErrors) { Set<NodeIdentifier> nodeSet = validationErrorMap.get(nodeValidationError); if (nodeSet == null) { nodeSet = new HashSet<>(); validationErrorMap.put(nodeValidationError, nodeSet); } nodeSet.add(nodeId); } } } /** * Normalizes the validation errors by prepending the corresponding nodes when the error does not exist across all nodes. * * @param validationErrorMap map * @param totalNodes total * @return normalized errors */ public Set<String> normalizedMergedValidationErrors(final Map<String, Set<NodeIdentifier>> validationErrorMap, int totalNodes) { final Set<String> normalizedValidationErrors = new HashSet<>(); for (final Map.Entry<String, Set<NodeIdentifier>> validationEntry : validationErrorMap.entrySet()) { final String msg = validationEntry.getKey(); final Set<NodeIdentifier> nodeIds = validationEntry.getValue(); if (nodeIds.size() == totalNodes) { normalizedValidationErrors.add(msg); } else { for (final NodeIdentifier nodeId : nodeIds) { normalizedValidationErrors .add(nodeId.getApiAddress() + ":" + nodeId.getApiPort() + " -- " + msg); } } } return normalizedValidationErrors; } /** * Merges the listing requests in the specified map into the specified listing request * * @param listingRequest the target listing request * @param listingRequestMap the mapping of all responses being merged */ private void mergeListingRequests(final ListingRequestDTO listingRequest, final Map<NodeIdentifier, ListingRequestDTO> listingRequestMap) { final Comparator<FlowFileSummaryDTO> comparator = new Comparator<FlowFileSummaryDTO>() { @Override public int compare(final FlowFileSummaryDTO dto1, final FlowFileSummaryDTO dto2) { int positionCompare = dto1.getPosition().compareTo(dto2.getPosition()); if (positionCompare != 0) { return positionCompare; } final String address1 = dto1.getClusterNodeAddress(); final String address2 = dto2.getClusterNodeAddress(); if (address1 == null && address2 == null) { return 0; } if (address1 == null) { return 1; } if (address2 == null) { return -1; } return address1.compareTo(address2); } }; final NavigableSet<FlowFileSummaryDTO> flowFileSummaries = new TreeSet<>(comparator); ListFlowFileState state = null; int numStepsCompleted = 0; int numStepsTotal = 0; int objectCount = 0; long byteCount = 0; boolean finished = true; for (final Map.Entry<NodeIdentifier, ListingRequestDTO> entry : listingRequestMap.entrySet()) { final NodeIdentifier nodeIdentifier = entry.getKey(); final String nodeAddress = nodeIdentifier.getApiAddress() + ":" + nodeIdentifier.getApiPort(); final ListingRequestDTO nodeRequest = entry.getValue(); numStepsTotal++; if (Boolean.TRUE.equals(nodeRequest.getFinished())) { numStepsCompleted++; } final QueueSizeDTO nodeQueueSize = nodeRequest.getQueueSize(); objectCount += nodeQueueSize.getObjectCount(); byteCount += nodeQueueSize.getByteCount(); if (!nodeRequest.getFinished()) { finished = false; } if (nodeRequest.getLastUpdated().after(listingRequest.getLastUpdated())) { listingRequest.setLastUpdated(nodeRequest.getLastUpdated()); } // Keep the state with the lowest ordinal value (the "least completed"). final ListFlowFileState nodeState = ListFlowFileState.valueOfDescription(nodeRequest.getState()); if (state == null || state.compareTo(nodeState) > 0) { state = nodeState; } if (nodeRequest.getFlowFileSummaries() != null) { for (final FlowFileSummaryDTO summaryDTO : nodeRequest.getFlowFileSummaries()) { summaryDTO.setClusterNodeId(nodeIdentifier.getId()); summaryDTO.setClusterNodeAddress(nodeAddress); flowFileSummaries.add(summaryDTO); // Keep the set from growing beyond our max if (flowFileSummaries.size() > listingRequest.getMaxResults()) { flowFileSummaries.pollLast(); } } } if (nodeRequest.getFailureReason() != null) { listingRequest.setFailureReason(nodeRequest.getFailureReason()); } } final List<FlowFileSummaryDTO> summaryDTOs = new ArrayList<>(flowFileSummaries); listingRequest.setFlowFileSummaries(summaryDTOs); final int percentCompleted = numStepsCompleted / numStepsTotal; listingRequest.setPercentCompleted(percentCompleted); listingRequest.setFinished(finished); listingRequest.getQueueSize().setByteCount(byteCount); listingRequest.getQueueSize().setObjectCount(objectCount); } /** * Merges the drop requests in the specified map into the specified drop request. * * @param dropRequest the target drop request * @param dropRequestMap the mapping of all responses being merged */ private void mergeDropRequests(final DropRequestDTO dropRequest, final Map<NodeIdentifier, DropRequestDTO> dropRequestMap) { boolean nodeWaiting = false; int originalCount = 0; long originalSize = 0; int currentCount = 0; long currentSize = 0; int droppedCount = 0; long droppedSize = 0; DropFlowFileState state = null; for (final Map.Entry<NodeIdentifier, DropRequestDTO> nodeEntry : dropRequestMap.entrySet()) { final DropRequestDTO nodeDropRequest = nodeEntry.getValue(); currentCount += nodeDropRequest.getCurrentCount(); currentSize += nodeDropRequest.getCurrentSize(); droppedCount += nodeDropRequest.getDroppedCount(); droppedSize += nodeDropRequest.getDroppedSize(); if (nodeDropRequest.getOriginalCount() == null) { nodeWaiting = true; } else { originalCount += nodeDropRequest.getOriginalCount(); originalSize += nodeDropRequest.getOriginalSize(); } final DropFlowFileState nodeState = DropFlowFileState.valueOfDescription(nodeDropRequest.getState()); if (state == null || state.compareTo(nodeState) > 0) { state = nodeState; } } dropRequest.setCurrentCount(currentCount); dropRequest.setCurrentSize(currentSize); dropRequest.setCurrent( FormatUtils.formatCount(currentCount) + " / " + FormatUtils.formatDataSize(currentSize)); dropRequest.setDroppedCount(droppedCount); dropRequest.setDroppedSize(droppedSize); dropRequest.setDropped( FormatUtils.formatCount(droppedCount) + " / " + FormatUtils.formatDataSize(droppedSize)); if (!nodeWaiting) { dropRequest.setOriginalCount(originalCount); dropRequest.setOriginalSize(originalSize); dropRequest.setOriginal( FormatUtils.formatCount(originalCount) + " / " + FormatUtils.formatDataSize(originalSize)); } if (state != null) { dropRequest.setState(state.toString()); } } // requires write lock to be already acquired unless request is not mutable private NodeResponse mergeResponses(final URI uri, final String method, final Set<NodeResponse> nodeResponses, final boolean mutableRequest) { // holds the one response of all the node responses to return to the client NodeResponse clientResponse = null; // holds the set of node responses that did not result in a 2XX response final Set<NodeResponse> problematicNodeResponses = new HashSet<>(); // map updated node to its response final Map<Node, NodeResponse> updatedNodesMap = new HashMap<>(); for (final Map.Entry<NodeResponse, Status> entry : httpResponseMapper.map(uri, nodeResponses).entrySet()) { final NodeResponse nodeResponse = entry.getKey(); final Status nodeStatus = entry.getValue(); // create new "updated" node by cloning old node and updating status final Node currentNode = getRawNode(nodeResponse.getNodeId().getId()); final Node updatedNode = currentNode.clone(); updatedNode.setStatus(nodeStatus); // map updated node to its response updatedNodesMap.put(updatedNode, nodeResponse); // record a client request and any requests that resulted in disconnection if (nodeStatus == Status.CONNECTED) { clientResponse = nodeResponse; } else if (nodeStatus == Status.DISCONNECTED) { problematicNodeResponses.add(nodeResponse); } } // determine if we have at least one response final boolean hasClientResponse = clientResponse != null; final boolean hasSuccessfulClientResponse = hasClientResponse && clientResponse.is2xx(); // drain the responses from the socket for those responses not being sent to the client final Set<NodeResponse> nodeResponsesToDrain = new HashSet<>(updatedNodesMap.values()); nodeResponsesToDrain.remove(clientResponse); if (hasSuccessfulClientResponse && isProcessorEndpoint(uri, method)) { final ProcessorEntity responseEntity = clientResponse.getClientResponse() .getEntity(ProcessorEntity.class); final ProcessorDTO processor = responseEntity.getProcessor(); final Map<NodeIdentifier, ProcessorDTO> processorMap = new HashMap<>(); for (final NodeResponse nodeResponse : updatedNodesMap.values()) { if (problematicNodeResponses.contains(nodeResponse)) { continue; } final ProcessorEntity nodeResponseEntity = nodeResponse == clientResponse ? responseEntity : nodeResponse.getClientResponse().getEntity(ProcessorEntity.class); final ProcessorDTO nodeProcessor = nodeResponseEntity.getProcessor(); processorMap.put(nodeResponse.getNodeId(), nodeProcessor); } mergeProcessorValidationErrors(processor, processorMap); clientResponse = new NodeResponse(clientResponse, responseEntity); } else if (hasSuccessfulClientResponse && isProcessorsEndpoint(uri, method)) { final ProcessorsEntity responseEntity = clientResponse.getClientResponse() .getEntity(ProcessorsEntity.class); final Set<ProcessorDTO> processors = responseEntity.getProcessors(); final Map<String, Map<NodeIdentifier, ProcessorDTO>> processorMap = new HashMap<>(); for (final NodeResponse nodeResponse : updatedNodesMap.values()) { if (problematicNodeResponses.contains(nodeResponse)) { continue; } final ProcessorsEntity nodeResponseEntity = nodeResponse == clientResponse ? responseEntity : nodeResponse.getClientResponse().getEntity(ProcessorsEntity.class); final Set<ProcessorDTO> nodeProcessors = nodeResponseEntity.getProcessors(); for (final ProcessorDTO nodeProcessor : nodeProcessors) { Map<NodeIdentifier, ProcessorDTO> innerMap = processorMap.get(nodeProcessor.getId()); if (innerMap == null) { innerMap = new HashMap<>(); processorMap.put(nodeProcessor.getId(), innerMap); } innerMap.put(nodeResponse.getNodeId(), nodeProcessor); } } for (final ProcessorDTO processor : processors) { final String procId = processor.getId(); final Map<NodeIdentifier, ProcessorDTO> mergeMap = processorMap.get(procId); mergeProcessorValidationErrors(processor, mergeMap); } // create a new client response clientResponse = new NodeResponse(clientResponse, responseEntity); } else if (hasSuccessfulClientResponse && isProcessGroupEndpoint(uri, method)) { final ProcessGroupEntity responseEntity = clientResponse.getClientResponse() .getEntity(ProcessGroupEntity.class); final ProcessGroupDTO responseDto = responseEntity.getProcessGroup(); final FlowSnippetDTO contents = responseDto.getContents(); if (contents == null) { if (!nodeResponsesToDrain.isEmpty()) { drainResponses(nodeResponsesToDrain); } } else { final Map<String, Map<NodeIdentifier, ProcessorDTO>> processorMap = new HashMap<>(); final Map<String, Map<NodeIdentifier, RemoteProcessGroupDTO>> remoteProcessGroupMap = new HashMap<>(); for (final NodeResponse nodeResponse : updatedNodesMap.values()) { if (problematicNodeResponses.contains(nodeResponse)) { continue; } final ProcessGroupEntity nodeResponseEntity = nodeResponse == clientResponse ? responseEntity : nodeResponse.getClientResponse().getEntity(ProcessGroupEntity.class); final ProcessGroupDTO nodeProcessGroup = nodeResponseEntity.getProcessGroup(); for (final ProcessorDTO nodeProcessor : nodeProcessGroup.getContents().getProcessors()) { Map<NodeIdentifier, ProcessorDTO> innerMap = processorMap.get(nodeProcessor.getId()); if (innerMap == null) { innerMap = new HashMap<>(); processorMap.put(nodeProcessor.getId(), innerMap); } innerMap.put(nodeResponse.getNodeId(), nodeProcessor); } for (final RemoteProcessGroupDTO nodeRemoteProcessGroup : nodeProcessGroup.getContents() .getRemoteProcessGroups()) { Map<NodeIdentifier, RemoteProcessGroupDTO> innerMap = remoteProcessGroupMap .get(nodeRemoteProcessGroup.getId()); if (innerMap == null) { innerMap = new HashMap<>(); remoteProcessGroupMap.put(nodeRemoteProcessGroup.getId(), innerMap); } innerMap.put(nodeResponse.getNodeId(), nodeRemoteProcessGroup); } } for (final ProcessorDTO processor : contents.getProcessors()) { final String procId = processor.getId(); final Map<NodeIdentifier, ProcessorDTO> mergeMap = processorMap.get(procId); mergeProcessorValidationErrors(processor, mergeMap); } for (final RemoteProcessGroupDTO remoteProcessGroup : contents.getRemoteProcessGroups()) { if (remoteProcessGroup.getContents() != null) { final String remoteProcessGroupId = remoteProcessGroup.getId(); final Map<NodeIdentifier, RemoteProcessGroupDTO> mergeMap = remoteProcessGroupMap .get(remoteProcessGroupId); mergeRemoteProcessGroup(remoteProcessGroup, mergeMap); } } } // create a new client response clientResponse = new NodeResponse(clientResponse, responseEntity); } else if (hasSuccessfulClientResponse && (isTemplateEndpoint(uri, method) || isFlowSnippetEndpoint(uri, method))) { final FlowSnippetEntity responseEntity = clientResponse.getClientResponse() .getEntity(FlowSnippetEntity.class); final FlowSnippetDTO contents = responseEntity.getContents(); if (contents == null) { if (!nodeResponsesToDrain.isEmpty()) { drainResponses(nodeResponsesToDrain); } } else { final Map<String, Map<NodeIdentifier, ProcessorDTO>> processorMap = new HashMap<>(); final Map<String, Map<NodeIdentifier, RemoteProcessGroupDTO>> remoteProcessGroupMap = new HashMap<>(); for (final NodeResponse nodeResponse : updatedNodesMap.values()) { if (problematicNodeResponses.contains(nodeResponse)) { continue; } final FlowSnippetEntity nodeResponseEntity = nodeResponse == clientResponse ? responseEntity : nodeResponse.getClientResponse().getEntity(FlowSnippetEntity.class); final FlowSnippetDTO nodeContents = nodeResponseEntity.getContents(); for (final ProcessorDTO nodeProcessor : nodeContents.getProcessors()) { Map<NodeIdentifier, ProcessorDTO> innerMap = processorMap.get(nodeProcessor.getId()); if (innerMap == null) { innerMap = new HashMap<>(); processorMap.put(nodeProcessor.getId(), innerMap); } innerMap.put(nodeResponse.getNodeId(), nodeProcessor); } for (final RemoteProcessGroupDTO nodeRemoteProcessGroup : nodeContents .getRemoteProcessGroups()) { Map<NodeIdentifier, RemoteProcessGroupDTO> innerMap = remoteProcessGroupMap .get(nodeRemoteProcessGroup.getId()); if (innerMap == null) { innerMap = new HashMap<>(); remoteProcessGroupMap.put(nodeRemoteProcessGroup.getId(), innerMap); } innerMap.put(nodeResponse.getNodeId(), nodeRemoteProcessGroup); } } for (final ProcessorDTO processor : contents.getProcessors()) { final String procId = processor.getId(); final Map<NodeIdentifier, ProcessorDTO> mergeMap = processorMap.get(procId); mergeProcessorValidationErrors(processor, mergeMap); } for (final RemoteProcessGroupDTO remoteProcessGroup : contents.getRemoteProcessGroups()) { if (remoteProcessGroup.getContents() != null) { final String remoteProcessGroupId = remoteProcessGroup.getId(); final Map<NodeIdentifier, RemoteProcessGroupDTO> mergeMap = remoteProcessGroupMap .get(remoteProcessGroupId); mergeRemoteProcessGroup(remoteProcessGroup, mergeMap); } } } // create a new client response clientResponse = new NodeResponse(clientResponse, responseEntity); } else if (hasSuccessfulClientResponse && isRemoteProcessGroupEndpoint(uri, method)) { final RemoteProcessGroupEntity responseEntity = clientResponse.getClientResponse() .getEntity(RemoteProcessGroupEntity.class); final RemoteProcessGroupDTO remoteProcessGroup = responseEntity.getRemoteProcessGroup(); final Map<NodeIdentifier, RemoteProcessGroupDTO> remoteProcessGroupMap = new HashMap<>(); for (final NodeResponse nodeResponse : updatedNodesMap.values()) { if (problematicNodeResponses.contains(nodeResponse)) { continue; } final RemoteProcessGroupEntity nodeResponseEntity = nodeResponse == clientResponse ? responseEntity : nodeResponse.getClientResponse().getEntity(RemoteProcessGroupEntity.class); final RemoteProcessGroupDTO nodeRemoteProcessGroup = nodeResponseEntity.getRemoteProcessGroup(); remoteProcessGroupMap.put(nodeResponse.getNodeId(), nodeRemoteProcessGroup); } mergeRemoteProcessGroup(remoteProcessGroup, remoteProcessGroupMap); clientResponse = new NodeResponse(clientResponse, responseEntity); } else if (hasSuccessfulClientResponse && isRemoteProcessGroupsEndpoint(uri, method)) { final RemoteProcessGroupsEntity responseEntity = clientResponse.getClientResponse() .getEntity(RemoteProcessGroupsEntity.class); final Set<RemoteProcessGroupDTO> remoteProcessGroups = responseEntity.getRemoteProcessGroups(); final Map<String, Map<NodeIdentifier, RemoteProcessGroupDTO>> remoteProcessGroupMap = new HashMap<>(); for (final NodeResponse nodeResponse : updatedNodesMap.values()) { if (problematicNodeResponses.contains(nodeResponse)) { continue; } final RemoteProcessGroupsEntity nodeResponseEntity = nodeResponse == clientResponse ? responseEntity : nodeResponse.getClientResponse().getEntity(RemoteProcessGroupsEntity.class); final Set<RemoteProcessGroupDTO> nodeRemoteProcessGroups = nodeResponseEntity .getRemoteProcessGroups(); for (final RemoteProcessGroupDTO nodeRemoteProcessGroup : nodeRemoteProcessGroups) { Map<NodeIdentifier, RemoteProcessGroupDTO> innerMap = remoteProcessGroupMap .get(nodeRemoteProcessGroup.getId()); if (innerMap == null) { innerMap = new HashMap<>(); remoteProcessGroupMap.put(nodeRemoteProcessGroup.getId(), innerMap); } innerMap.put(nodeResponse.getNodeId(), nodeRemoteProcessGroup); } } for (final RemoteProcessGroupDTO remoteProcessGroup : remoteProcessGroups) { final String remoteProcessGroupId = remoteProcessGroup.getId(); final Map<NodeIdentifier, RemoteProcessGroupDTO> mergeMap = remoteProcessGroupMap .get(remoteProcessGroupId); mergeRemoteProcessGroup(remoteProcessGroup, mergeMap); } // create a new client response clientResponse = new NodeResponse(clientResponse, responseEntity); } else if (hasSuccessfulClientResponse && isProvenanceQueryEndpoint(uri, method)) { final ProvenanceEntity responseEntity = clientResponse.getClientResponse() .getEntity(ProvenanceEntity.class); final ProvenanceDTO query = responseEntity.getProvenance(); final Map<NodeIdentifier, ProvenanceDTO> resultsMap = new HashMap<>(); for (final NodeResponse nodeResponse : updatedNodesMap.values()) { if (problematicNodeResponses.contains(nodeResponse)) { continue; } final ProvenanceEntity nodeResponseEntity = nodeResponse == clientResponse ? responseEntity : nodeResponse.getClientResponse().getEntity(ProvenanceEntity.class); final ProvenanceDTO nodeQuery = nodeResponseEntity.getProvenance(); resultsMap.put(nodeResponse.getNodeId(), nodeQuery); } mergeProvenanceQueryResults(query, resultsMap, problematicNodeResponses); clientResponse = new NodeResponse(clientResponse, responseEntity); } else if (hasSuccessfulClientResponse && isProvenanceEventEndpoint(uri, method)) { final ProvenanceEventEntity responseEntity = clientResponse.getClientResponse() .getEntity(ProvenanceEventEntity.class); final ProvenanceEventDTO event = responseEntity.getProvenanceEvent(); // this request was sent to a specific node... populate its details final NodeIdentifier nodeId = clientResponse.getNodeId(); event.setClusterNodeId(nodeId.getId()); event.setClusterNodeAddress(nodeId.getApiAddress() + ":" + nodeId.getApiPort()); clientResponse = new NodeResponse(clientResponse, responseEntity); } else if (hasSuccessfulClientResponse && isControllerServiceEndpoint(uri, method)) { final ControllerServiceEntity responseEntity = clientResponse.getClientResponse() .getEntity(ControllerServiceEntity.class); final ControllerServiceDTO controllerService = responseEntity.getControllerService(); final Map<NodeIdentifier, ControllerServiceDTO> resultsMap = new HashMap<>(); for (final NodeResponse nodeResponse : updatedNodesMap.values()) { if (problematicNodeResponses.contains(nodeResponse)) { continue; } final ControllerServiceEntity nodeResponseEntity = nodeResponse == clientResponse ? responseEntity : nodeResponse.getClientResponse().getEntity(ControllerServiceEntity.class); final ControllerServiceDTO nodeControllerService = nodeResponseEntity.getControllerService(); resultsMap.put(nodeResponse.getNodeId(), nodeControllerService); } mergeControllerService(controllerService, resultsMap); clientResponse = new NodeResponse(clientResponse, responseEntity); } else if (hasSuccessfulClientResponse && isControllerServicesEndpoint(uri, method)) { final ControllerServicesEntity responseEntity = clientResponse.getClientResponse() .getEntity(ControllerServicesEntity.class); final Set<ControllerServiceDTO> controllerServices = responseEntity.getControllerServices(); final Map<String, Map<NodeIdentifier, ControllerServiceDTO>> controllerServiceMap = new HashMap<>(); for (final NodeResponse nodeResponse : updatedNodesMap.values()) { if (problematicNodeResponses.contains(nodeResponse)) { continue; } final ControllerServicesEntity nodeResponseEntity = nodeResponse == clientResponse ? responseEntity : nodeResponse.getClientResponse().getEntity(ControllerServicesEntity.class); final Set<ControllerServiceDTO> nodeControllerServices = nodeResponseEntity.getControllerServices(); for (final ControllerServiceDTO nodeControllerService : nodeControllerServices) { Map<NodeIdentifier, ControllerServiceDTO> innerMap = controllerServiceMap .get(nodeControllerService.getId()); if (innerMap == null) { innerMap = new HashMap<>(); controllerServiceMap.put(nodeControllerService.getId(), innerMap); } innerMap.put(nodeResponse.getNodeId(), nodeControllerService); } } for (final ControllerServiceDTO controllerService : controllerServices) { final String procId = controllerService.getId(); final Map<NodeIdentifier, ControllerServiceDTO> mergeMap = controllerServiceMap.get(procId); mergeControllerService(controllerService, mergeMap); } // create a new client response clientResponse = new NodeResponse(clientResponse, responseEntity); } else if (hasSuccessfulClientResponse && isControllerServiceReferenceEndpoint(uri, method)) { final ControllerServiceReferencingComponentsEntity responseEntity = clientResponse.getClientResponse() .getEntity(ControllerServiceReferencingComponentsEntity.class); final Set<ControllerServiceReferencingComponentDTO> referencingComponents = responseEntity .getControllerServiceReferencingComponents(); final Map<NodeIdentifier, Set<ControllerServiceReferencingComponentDTO>> resultsMap = new HashMap<>(); for (final NodeResponse nodeResponse : updatedNodesMap.values()) { if (problematicNodeResponses.contains(nodeResponse)) { continue; } final ControllerServiceReferencingComponentsEntity nodeResponseEntity = nodeResponse == clientResponse ? responseEntity : nodeResponse.getClientResponse() .getEntity(ControllerServiceReferencingComponentsEntity.class); final Set<ControllerServiceReferencingComponentDTO> nodeReferencingComponents = nodeResponseEntity .getControllerServiceReferencingComponents(); resultsMap.put(nodeResponse.getNodeId(), nodeReferencingComponents); } mergeControllerServiceReferences(referencingComponents, resultsMap); clientResponse = new NodeResponse(clientResponse, responseEntity); } else if (hasSuccessfulClientResponse && isReportingTaskEndpoint(uri, method)) { final ReportingTaskEntity responseEntity = clientResponse.getClientResponse() .getEntity(ReportingTaskEntity.class); final ReportingTaskDTO reportingTask = responseEntity.getReportingTask(); final Map<NodeIdentifier, ReportingTaskDTO> resultsMap = new HashMap<>(); for (final NodeResponse nodeResponse : updatedNodesMap.values()) { if (problematicNodeResponses.contains(nodeResponse)) { continue; } final ReportingTaskEntity nodeResponseEntity = nodeResponse == clientResponse ? responseEntity : nodeResponse.getClientResponse().getEntity(ReportingTaskEntity.class); final ReportingTaskDTO nodeReportingTask = nodeResponseEntity.getReportingTask(); resultsMap.put(nodeResponse.getNodeId(), nodeReportingTask); } mergeReportingTask(reportingTask, resultsMap); clientResponse = new NodeResponse(clientResponse, responseEntity); } else if (hasSuccessfulClientResponse && isReportingTasksEndpoint(uri, method)) { final ReportingTasksEntity responseEntity = clientResponse.getClientResponse() .getEntity(ReportingTasksEntity.class); final Set<ReportingTaskDTO> reportingTaskSet = responseEntity.getReportingTasks(); final Map<String, Map<NodeIdentifier, ReportingTaskDTO>> reportingTaskMap = new HashMap<>(); for (final NodeResponse nodeResponse : updatedNodesMap.values()) { if (problematicNodeResponses.contains(nodeResponse)) { continue; } final ReportingTasksEntity nodeResponseEntity = nodeResponse == clientResponse ? responseEntity : nodeResponse.getClientResponse().getEntity(ReportingTasksEntity.class); final Set<ReportingTaskDTO> nodeReportingTasks = nodeResponseEntity.getReportingTasks(); for (final ReportingTaskDTO nodeReportingTask : nodeReportingTasks) { Map<NodeIdentifier, ReportingTaskDTO> innerMap = reportingTaskMap .get(nodeReportingTask.getId()); if (innerMap == null) { innerMap = new HashMap<>(); reportingTaskMap.put(nodeReportingTask.getId(), innerMap); } innerMap.put(nodeResponse.getNodeId(), nodeReportingTask); } } for (final ReportingTaskDTO reportingTask : reportingTaskSet) { final String procId = reportingTask.getId(); final Map<NodeIdentifier, ReportingTaskDTO> mergeMap = reportingTaskMap.get(procId); mergeReportingTask(reportingTask, mergeMap); } // create a new client response clientResponse = new NodeResponse(clientResponse, responseEntity); } else if (hasSuccessfulClientResponse && isDropRequestEndpoint(uri, method)) { final DropRequestEntity responseEntity = clientResponse.getClientResponse() .getEntity(DropRequestEntity.class); final DropRequestDTO dropRequest = responseEntity.getDropRequest(); final Map<NodeIdentifier, DropRequestDTO> resultsMap = new HashMap<>(); for (final NodeResponse nodeResponse : updatedNodesMap.values()) { if (problematicNodeResponses.contains(nodeResponse)) { continue; } final DropRequestEntity nodeResponseEntity = nodeResponse == clientResponse ? responseEntity : nodeResponse.getClientResponse().getEntity(DropRequestEntity.class); final DropRequestDTO nodeDropRequest = nodeResponseEntity.getDropRequest(); resultsMap.put(nodeResponse.getNodeId(), nodeDropRequest); } mergeDropRequests(dropRequest, resultsMap); clientResponse = new NodeResponse(clientResponse, responseEntity); } else if (hasSuccessfulClientResponse && isListFlowFilesEndpoint(uri, method)) { final ListingRequestEntity responseEntity = clientResponse.getClientResponse() .getEntity(ListingRequestEntity.class); final ListingRequestDTO listingRequest = responseEntity.getListingRequest(); final Map<NodeIdentifier, ListingRequestDTO> resultsMap = new HashMap<>(); for (final NodeResponse nodeResponse : updatedNodesMap.values()) { if (problematicNodeResponses.contains(nodeResponse)) { continue; } final ListingRequestEntity nodeResponseEntity = nodeResponse == clientResponse ? responseEntity : nodeResponse.getClientResponse().getEntity(ListingRequestEntity.class); final ListingRequestDTO nodeListingRequest = nodeResponseEntity.getListingRequest(); resultsMap.put(nodeResponse.getNodeId(), nodeListingRequest); } mergeListingRequests(listingRequest, resultsMap); clientResponse = new NodeResponse(clientResponse, responseEntity); } else { if (!nodeResponsesToDrain.isEmpty()) { drainResponses(nodeResponsesToDrain); } } /* * Nodes that encountered issues handling the request are marked as * disconnected for mutable requests (e.g., post, put, delete). For * other requests (e.g., get, head), the nodes remain in their current * state even if they had problems handling the request. */ if (mutableRequest) { // all nodes failed final boolean allNodesFailed = problematicNodeResponses.size() == nodeResponses.size(); // some nodes had a problematic response because of a missing counter, ensure the are not disconnected final boolean someNodesFailedMissingCounter = !problematicNodeResponses.isEmpty() && problematicNodeResponses.size() < nodeResponses.size() && isMissingCounter(problematicNodeResponses, uri); // ensure nodes stay connected in certain scenarios if (allNodesFailed || someNodesFailedMissingCounter) { for (final Map.Entry<Node, NodeResponse> updatedNodeEntry : updatedNodesMap.entrySet()) { final NodeResponse nodeResponse = updatedNodeEntry.getValue(); final Node node = updatedNodeEntry.getKey(); if (problematicNodeResponses.contains(nodeResponse)) { node.setStatus(Status.CONNECTED); problematicNodeResponses.remove(nodeResponse); } } } // set the updated nodes nodes.removeAll(updatedNodesMap.keySet()); nodes.addAll(updatedNodesMap.keySet()); // notify service of updated node set notifyDataFlowManagementServiceOfNodeStatusChange(); // mark flow as stale since this request could have changed the flow notifyDataFlowManagmentServiceOfFlowStateChange(PersistedFlowState.STALE); // disconnect problematic nodes if (!problematicNodeResponses.isEmpty()) { if (problematicNodeResponses.size() < nodeResponses.size()) { logger.warn(String.format( "One or more nodes failed to process URI '%s'. Requesting each node to disconnect from cluster.", uri)); disconnectNodes(problematicNodeResponses, "Failed to process URI " + uri); } else { logger.warn( "All nodes failed to process URI {}. As a result, no node will be disconnected from cluster", uri); } } } return clientResponse; } /** * Determines if all problematic responses were due to 404 NOT_FOUND. Assumes that problematicNodeResponses is not empty and is not comprised of responses from all nodes in the cluster (at least * one node contained the counter in question). * * @param problematicNodeResponses The problematic node responses * @param uri The URI for the request * @return Whether all problematic node responses were due to a missing counter */ private boolean isMissingCounter(final Set<NodeResponse> problematicNodeResponses, final URI uri) { if (isCountersEndpoint(uri)) { boolean notFound = true; for (final NodeResponse problematicResponse : problematicNodeResponses) { if (problematicResponse.getStatus() != 404) { notFound = false; break; } } return notFound; } return false; } /** * Drains the node responses off of the socket to ensure that the socket is appropriately cleaned-up. * * @param nodeResponses the collection of node responses */ private void drainResponses(final Collection<NodeResponse> nodeResponses) { // fail fast if nothing to do if (nodeResponses.isEmpty()) { return; } final ExecutorService executorService = Executors .newFixedThreadPool(properties.getClusterManagerProtocolThreads()); final CompletionService<Void> completionService = new ExecutorCompletionService<>(executorService); for (final NodeResponse nodeResponse : nodeResponses) { // if we received a response, then clear out the response data if (!nodeResponse.hasThrowable()) { completionService.submit(new Runnable() { @Override public void run() { try { try (final OutputStream drain = new OutputStream() { @Override public void write(final int b) { /* drain response */ } }) { ((StreamingOutput) nodeResponse.getResponse().getEntity()).write(drain); } } catch (final IOException | WebApplicationException ex) { logger.info("Failed clearing out non-client response buffer due to: " + ex, ex); } } }, null); } } executorService.shutdown(); } /** * A helper method to disconnect nodes that returned unsuccessful HTTP responses because of a replicated request. Disconnection requests are sent concurrently. * */ private void disconnectNodes(final Set<NodeResponse> nodeResponses, final String explanation) { // return fast if nothing to do if (nodeResponses == null || nodeResponses.isEmpty()) { return; } final ExecutorService executorService = Executors .newFixedThreadPool(properties.getClusterManagerProtocolThreads()); final CompletionService<Void> completionService = new ExecutorCompletionService<>(executorService); for (final NodeResponse nodeResponse : nodeResponses) { completionService.submit(new Runnable() { @Override public void run() { final NodeIdentifier nodeId = nodeResponse.getNodeId(); final int responseStatus = nodeResponse.getStatus(); final URI requestUri = nodeResponse.getRequestUri(); final StringBuilder msgBuilder = new StringBuilder(); msgBuilder.append("Requesting disconnection for node ").append(nodeId) .append(" for request URI ").append(requestUri); if (nodeResponse.hasThrowable()) { msgBuilder.append(" because manager encountered exception when issuing request: ") .append(nodeResponse.getThrowable()); // log stack trace anytime we have a throwable ((NiFiLog) logger).getWrappedLog().info(msgBuilder.toString(), nodeResponse.getThrowable()); addEvent(nodeId, "Manager encountered exception when issuing request for URI " + requestUri); addBulletin(nodeId, Severity.ERROR, "Manager encountered exception when issuing request for URI " + requestUri + "; node will be disconnected"); } else { msgBuilder.append(" because HTTP response status was ").append(responseStatus); logger.info(msgBuilder.toString()); addEvent(nodeId, "HTTP response status was unsuccessful (" + responseStatus + ") for request URI " + requestUri); addBulletin(nodeId, Severity.ERROR, "HTTP response status was unsuccessful (" + responseStatus + ") for request URI " + requestUri); } requestDisconnectionQuietly(nodeId, explanation); } }, null); } executorService.shutdown(); } /** * Returns false if an internal protocol message was received by a node listed in the firewall. If no firewall is configured, then false is always returned. * * @param ip the IP of the remote machine * * @return false if the IP is listed in the firewall or if the firewall is not configured; true otherwise */ private boolean isBlockedByFirewall(final String ip) { if (isFirewallConfigured()) { return !clusterFirewall.isPermissible(ip); } else { return false; } } private Set<Node> getRawNodes(final Status... statuses) { readLock.lock(); try { final Set<Node> result = new HashSet<>(); if (statuses == null || statuses.length == 0) { result.addAll(nodes); } else { for (final Node node : nodes) { for (final Node.Status status : statuses) { if (node.getStatus() == status) { result.add(node); break; } } } } return result; } finally { readLock.unlock("getRawNodes(Status...)"); } } private Node getRawNode(final String nodeId) { readLock.lock(); try { for (final Node node : nodes) { if (node.getNodeId().getId().equals(nodeId)) { return node; } } return null; } finally { readLock.unlock("getRawNode(String)"); } } /** * Resolves a proposed node identifier to a node identifier that the manager approves. If the proposed node identifier conflicts with an existing node identifier, then an approved node identifier * is generated and returned to the caller. * * @param proposedNodeId a proposed identifier * * @return the node identifier that should be used */ private NodeIdentifier resolveProposedNodeIdentifier(final NodeIdentifier proposedNodeId) { readLock.lock(); try { for (final Node node : nodes) { final NodeIdentifier nodeId = node.getNodeId(); // are the ids the same final boolean sameId = nodeId.equals(proposedNodeId); // are the service coordinates the same final boolean sameServiceCoordinates = nodeId.logicallyEquals(proposedNodeId); if (sameId && sameServiceCoordinates) { // we know about this node and it has the same ID, so the proposal is fine return proposedNodeId; } else if (sameId && !sameServiceCoordinates) { // proposed ID conflicts with existing node ID, so assign a new ID final NodeIdentifier resolvedIdentifier = new NodeIdentifier(UUID.randomUUID().toString(), proposedNodeId.getApiAddress(), proposedNodeId.getApiPort(), proposedNodeId.getSocketAddress(), proposedNodeId.getSocketPort()); logger.info(String.format( "Using Node Identifier %s because proposed node identifier %s conflicts existing node identifiers", resolvedIdentifier, proposedNodeId)); return resolvedIdentifier; } else if (!sameId && sameServiceCoordinates) { // we know about this node, so we'll use the existing ID logger.debug(String.format( "Using Node Identifier %s because proposed node identifier %s matches the service coordinates", nodeId, proposedNodeId)); return nodeId; } } // proposal does not conflict with existing nodes return proposedNodeId; } finally { readLock.unlock("resolveProposedNodeIdentifier"); } } private boolean isHeartbeatMonitorRunning() { readLock.lock(); try { return heartbeatMonitor != null; } finally { readLock.unlock("isHeartbeatMonitorRunning"); } } private boolean canChangeNodeState(final String method, final URI uri) { return HttpMethod.DELETE.equalsIgnoreCase(method) || HttpMethod.POST.equalsIgnoreCase(method) || HttpMethod.PUT.equalsIgnoreCase(method); } private void notifyDataFlowManagementServiceOfNodeStatusChange() { writeLock.lock(); try { // tell service about the currently connected nodes logger.debug("Notifying DataFlow Management Service of current set of connected nodes."); dataFlowManagementService.setNodeIds(getNodeIds(Status.CONNECTED)); } finally { writeLock.unlock("notifyDataFlowManagementServiceOfNodeStatusChange"); } } private void notifyDataFlowManagmentServiceOfFlowStateChange(final PersistedFlowState newState) { writeLock.lock(); try { logger.debug("Notifying DataFlow Management Service that flow state is " + newState); dataFlowManagementService.setPersistedFlowState(newState); if (newState != PersistedFlowState.CURRENT) { cachedDataFlow = null; /* do not reset primary node ID because only the data flow has changed */ } } finally { writeLock.unlock("notifyDataFlowManagementServiceOfFlowStateChange"); } } private void logNodes(final String header, final Logger logger) { if (logger.isTraceEnabled()) { if (StringUtils.isNotBlank(header)) { logger.trace(header); } for (final Node node : getNodes()) { logger.trace(node.getNodeId() + " : " + node.getStatus()); } } } private void executeSafeModeTask() { new Thread(new Runnable() { private final long threadStartTime = System.currentTimeMillis(); @Override public void run() { logger.info("Entering safe mode..."); final int safeModeSeconds = (int) FormatUtils .getTimeDuration(properties.getClusterManagerSafeModeDuration(), TimeUnit.SECONDS); final long timeToElect = safeModeSeconds <= 0 ? Long.MAX_VALUE : threadStartTime + TimeUnit.MILLISECONDS.convert(safeModeSeconds, TimeUnit.SECONDS); boolean exitSafeMode = false; while (isRunning()) { writeLock.lock(); try { final long currentTime = System.currentTimeMillis(); if (timeToElect < currentTime) { final Set<NodeIdentifier> connectedNodeIds = getNodeIds(Status.CONNECTED); if (!connectedNodeIds.isEmpty()) { // get first connected node ID final NodeIdentifier connectedNodeId = connectedNodeIds.iterator().next(); if (assignPrimaryRole(connectedNodeId)) { try { setPrimaryNodeId(connectedNodeId); exitSafeMode = true; } catch (final DaoException de) { final String message = String.format( "Failed to persist primary node ID '%s' in cluster dataflow.", connectedNodeId); logger.warn(message); addBulletin(connectedNodeId, Severity.WARNING, message); revokePrimaryRole(connectedNodeId); } } } } if (!isInSafeMode()) { // a primary node has been selected outside of this thread exitSafeMode = true; logger.info("Exiting safe mode because " + primaryNodeId + " has been assigned the primary role."); break; } } finally { writeLock.unlock("executeSafeModeTask"); } if (!exitSafeMode) { // sleep for a bit try { Thread.sleep(1000); } catch (final InterruptedException ie) { return; } } } } }).start(); } /** * This timer task simply processes any pending heartbeats. This timer task is not strictly needed, as HeartbeatMonitoringTimerTask will do this. However, this task is scheduled much more * frequently and by processing the heartbeats more frequently, the stats that we report have less of a delay. */ private class ProcessPendingHeartbeatsTask extends TimerTask { @Override public void run() { writeLock.lock(); try { processPendingHeartbeats(); } finally { writeLock.unlock("Process Pending Heartbeats Task"); } } } /** * A timer task to detect nodes that have not sent a heartbeat in a while. The "problem" nodes are marked as disconnected due to lack of heartbeat by the task. No disconnection request is sent to * the node. This is because either the node is not functioning in which case sending the request is futile or the node is running a bit slow. In the latter case, we'll wait for the next heartbeat * and send a reconnection request when we process the heartbeat in the heartbeatHandler() method. */ private class HeartbeatMonitoringTimerTask extends TimerTask { @Override public void run() { // keep track of any status changes boolean statusChanged = false; writeLock.lock(); try { // process all of the heartbeats before we decided to kick anyone out of the cluster. logger.debug("Processing pending heartbeats..."); processPendingHeartbeats(); logger.debug("Executing heartbeat monitoring"); // check for any nodes that have not heartbeated in a long time for (final Node node : getRawNodes(Status.CONNECTED)) { // return prematurely if we were interrupted if (Thread.currentThread().isInterrupted()) { return; } // check if we received a recent heartbeat, changing status to disconnected if necessary final long lastHeardTimestamp = node.getHeartbeat().getCreatedTimestamp(); final int heartbeatGapSeconds = (int) (new Date().getTime() - lastHeardTimestamp) / 1000; if (heartbeatGapSeconds > getMaxHeartbeatGapSeconds()) { node.setHeartbeatDisconnection(); addEvent(node.getNodeId(), "Node disconnected due to lack of heartbeat."); addBulletin(node, Severity.WARNING, "Node disconnected due to lack of heartbeat"); statusChanged = true; } } // if a status change occurred, make the necessary updates if (statusChanged) { logNodes("Heartbeat Monitoring disconnected node(s)", logger); // notify service of updated node set notifyDataFlowManagementServiceOfNodeStatusChange(); } else { logNodes("Heartbeat Monitoring determined all nodes are healthy", logger); } } catch (final Exception ex) { logger.warn("Heartbeat monitor experienced exception while monitoring: " + ex, ex); } finally { writeLock.unlock("HeartbeatMonitoringTimerTask"); } } } @Override public ClusterNodeInformation getNodeInformation() { readLock.lock(); try { final Collection<NodeInformation> nodeInfos = new ArrayList<>(); for (final Node node : getRawNodes(Status.CONNECTED)) { final NodeIdentifier id = node.getNodeId(); final HeartbeatPayload heartbeat = node.getHeartbeatPayload(); if (heartbeat == null) { continue; } final Integer siteToSitePort = heartbeat.getSiteToSitePort(); if (siteToSitePort == null) { continue; } final int flowFileCount = (int) heartbeat.getTotalFlowFileCount(); final NodeInformation nodeInfo = new NodeInformation(id.getApiAddress(), siteToSitePort, id.getApiPort(), heartbeat.isSiteToSiteSecure(), flowFileCount); nodeInfos.add(nodeInfo); } final ClusterNodeInformation clusterNodeInfo = new ClusterNodeInformation(); clusterNodeInfo.setNodeInformation(nodeInfos); return clusterNodeInfo; } finally { readLock.unlock("getNodeInformation"); } } @Override public BulletinRepository getBulletinRepository() { return bulletinRepository; } @Override public ProcessGroupStatus getProcessGroupStatus(final String groupId) { final Set<Node> connectedNodes = getNodes(Node.Status.CONNECTED); // ensure there are some nodes in the cluster if (connectedNodes.isEmpty()) { throw new NoConnectedNodesException(); } ProcessGroupStatus mergedProcessGroupStatus = null; for (final Node node : connectedNodes) { final NodeIdentifier nodeId = node.getNodeId(); final HeartbeatPayload nodeHeartbeatPayload = node.getHeartbeatPayload(); if (nodeHeartbeatPayload == null) { continue; } final ProcessGroupStatus nodeRootProcessGroupStatus = nodeHeartbeatPayload.getProcessGroupStatus(); final ProcessGroupStatus nodeProcessGroupStatus = groupId.equals(ROOT_GROUP_ID_ALIAS) ? nodeRootProcessGroupStatus : getProcessGroupStatus(nodeRootProcessGroupStatus, groupId); if (nodeProcessGroupStatus == null) { continue; } if (mergedProcessGroupStatus == null) { mergedProcessGroupStatus = nodeProcessGroupStatus.clone(); // update any issues with the node label if (mergedProcessGroupStatus.getRemoteProcessGroupStatus() != null) { for (final RemoteProcessGroupStatus remoteProcessGroupStatus : mergedProcessGroupStatus .getRemoteProcessGroupStatus()) { final List<String> nodeAuthorizationIssues = remoteProcessGroupStatus .getAuthorizationIssues(); if (!nodeAuthorizationIssues.isEmpty()) { for (final ListIterator<String> iter = nodeAuthorizationIssues.listIterator(); iter .hasNext();) { final String Issue = iter.next(); iter.set( "[" + nodeId.getApiAddress() + ":" + nodeId.getApiPort() + "] -- " + Issue); } remoteProcessGroupStatus.setAuthorizationIssues(nodeAuthorizationIssues); } } } } else { final ProcessGroupStatus nodeClone = nodeProcessGroupStatus.clone(); for (final RemoteProcessGroupStatus remoteProcessGroupStatus : nodeClone .getRemoteProcessGroupStatus()) { final List<String> nodeAuthorizationIssues = remoteProcessGroupStatus.getAuthorizationIssues(); if (!nodeAuthorizationIssues.isEmpty()) { for (final ListIterator<String> iter = nodeAuthorizationIssues.listIterator(); iter .hasNext();) { final String Issue = iter.next(); iter.set("[" + nodeId.getApiAddress() + ":" + nodeId.getApiPort() + "] -- " + Issue); } remoteProcessGroupStatus.setAuthorizationIssues(nodeAuthorizationIssues); } } ProcessGroupStatus.merge(mergedProcessGroupStatus, nodeClone); } } return mergedProcessGroupStatus; } private ProcessGroupStatus getProcessGroupStatus(final ProcessGroupStatus parent, final String groupId) { if (parent.getId().equals(groupId)) { return parent; } for (final ProcessGroupStatus child : parent.getProcessGroupStatus()) { final ProcessGroupStatus matching = getProcessGroupStatus(child, groupId); if (matching != null) { return matching; } } return null; } @Override public SystemDiagnostics getSystemDiagnostics() { final Set<Node> connectedNodes = getNodes(Node.Status.CONNECTED); // ensure there are some nodes... if (connectedNodes.isEmpty()) { throw new NoConnectedNodesException(); } SystemDiagnostics clusterDiagnostics = null; for (final Node node : connectedNodes) { final HeartbeatPayload nodeHeartbeatPayload = node.getHeartbeatPayload(); if (nodeHeartbeatPayload == null) { continue; } final SystemDiagnostics nodeDiagnostics = nodeHeartbeatPayload.getSystemDiagnostics(); if (nodeDiagnostics == null) { continue; } if (clusterDiagnostics == null) { clusterDiagnostics = nodeDiagnostics.clone(); } else { merge(clusterDiagnostics, nodeDiagnostics); } } return clusterDiagnostics; } private void merge(final SystemDiagnostics target, final SystemDiagnostics sd) { // threads target.setDaemonThreads(target.getDaemonThreads() + sd.getDaemonThreads()); target.setTotalThreads(target.getTotalThreads() + sd.getTotalThreads()); // heap target.setTotalHeap(target.getTotalHeap() + sd.getTotalHeap()); target.setUsedHeap(target.getUsedHeap() + sd.getUsedHeap()); target.setMaxHeap(target.getMaxHeap() + sd.getMaxHeap()); // non heap target.setTotalNonHeap(target.getTotalNonHeap() + sd.getTotalNonHeap()); target.setUsedNonHeap(target.getUsedNonHeap() + sd.getUsedNonHeap()); target.setMaxNonHeap(target.getMaxNonHeap() + sd.getMaxNonHeap()); // processors target.setAvailableProcessors(target.getAvailableProcessors() + sd.getAvailableProcessors()); // load if (sd.getProcessorLoadAverage() != null) { if (target.getProcessorLoadAverage() != null) { target.setProcessorLoadAverage(target.getProcessorLoadAverage() + sd.getProcessorLoadAverage()); } else { target.setProcessorLoadAverage(sd.getProcessorLoadAverage()); } } // db disk usage merge(target.getFlowFileRepositoryStorageUsage(), sd.getFlowFileRepositoryStorageUsage()); // repo disk usage final Map<String, StorageUsage> targetContentRepoMap; if (target.getContentRepositoryStorageUsage() == null) { targetContentRepoMap = new LinkedHashMap<>(); target.setContentRepositoryStorageUsage(targetContentRepoMap); } else { targetContentRepoMap = target.getContentRepositoryStorageUsage(); } if (sd.getContentRepositoryStorageUsage() != null) { for (final Map.Entry<String, StorageUsage> sdEntry : sd.getContentRepositoryStorageUsage().entrySet()) { final StorageUsage mergedDiskUsage = targetContentRepoMap.get(sdEntry.getKey()); if (mergedDiskUsage == null) { targetContentRepoMap.put(sdEntry.getKey(), sdEntry.getValue()); } else { merge(mergedDiskUsage, sdEntry.getValue()); } } } // garbage collection final Map<String, GarbageCollection> targetGarbageCollection; if (target.getGarbageCollection() == null) { targetGarbageCollection = new LinkedHashMap<>(); target.setGarbageCollection(targetGarbageCollection); } else { targetGarbageCollection = target.getGarbageCollection(); } if (sd.getGarbageCollection() != null) { for (final Map.Entry<String, GarbageCollection> gcEntry : sd.getGarbageCollection().entrySet()) { final GarbageCollection mergedGarbageCollection = targetGarbageCollection.get(gcEntry.getKey()); if (mergedGarbageCollection == null) { targetGarbageCollection.put(gcEntry.getKey(), gcEntry.getValue().clone()); } else { merge(mergedGarbageCollection, gcEntry.getValue()); } } } } private void merge(final StorageUsage target, final StorageUsage du) { target.setFreeSpace(target.getFreeSpace() + du.getFreeSpace()); target.setTotalSpace(target.getTotalSpace() + du.getTotalSpace()); } private void merge(final GarbageCollection target, final GarbageCollection gc) { target.setCollectionCount(target.getCollectionCount() + gc.getCollectionCount()); target.setCollectionTime(target.getCollectionTime() + gc.getCollectionTime()); } public static Date normalizeStatusSnapshotDate(final Date toNormalize, final long numMillis) { final long time = toNormalize.getTime(); return new Date(time - time % numMillis); } private NodeDTO createNodeDTO(final Node node) { final NodeDTO nodeDto = new NodeDTO(); final NodeIdentifier nodeId = node.getNodeId(); nodeDto.setNodeId(nodeId.getId()); nodeDto.setAddress(nodeId.getApiAddress()); nodeDto.setApiPort(nodeId.getApiPort()); nodeDto.setStatus(node.getStatus().name()); nodeDto.setPrimary(node.equals(getPrimaryNode())); final Date connectionRequested = new Date(node.getConnectionRequestedTimestamp()); nodeDto.setConnectionRequested(connectionRequested); return nodeDto; } private List<StatusSnapshotDTO> aggregate(Map<Date, List<StatusSnapshot>> snapshotsToAggregate) { // Aggregate the snapshots final List<StatusSnapshotDTO> aggregatedSnapshotDtos = new ArrayList<>(); for (final Map.Entry<Date, List<StatusSnapshot>> entry : snapshotsToAggregate.entrySet()) { final List<StatusSnapshot> snapshots = entry.getValue(); final StatusSnapshot reducedSnapshot = snapshots.get(0).getValueReducer().reduce(snapshots); final StatusSnapshotDTO dto = new StatusSnapshotDTO(); dto.setTimestamp(reducedSnapshot.getTimestamp()); dto.setStatusMetrics(StatusHistoryUtil.createStatusSnapshotDto(reducedSnapshot).getStatusMetrics()); aggregatedSnapshotDtos.add(dto); } return aggregatedSnapshotDtos; } public ClusterStatusHistoryDTO getProcessorStatusHistory(final String processorId) { return getProcessorStatusHistory(processorId, null, null, Integer.MAX_VALUE); } public ClusterStatusHistoryDTO getProcessorStatusHistory(final String processorId, final Date startDate, final Date endDate, final int preferredDataPoints) { final List<NodeStatusHistoryDTO> nodeHistories = new ArrayList<>(); StatusHistoryDTO lastStatusHistory = null; final Set<MetricDescriptor<?>> processorDescriptors = new LinkedHashSet<>(); final Map<Date, List<StatusSnapshot>> snapshotsToAggregate = new TreeMap<>(); for (final Node node : getRawNodes()) { final ComponentStatusRepository statusRepository = componentMetricsRepositoryMap.get(node.getNodeId()); if (statusRepository == null) { continue; } final StatusHistory statusHistory = statusRepository.getProcessorStatusHistory(processorId, startDate, endDate, preferredDataPoints); if (statusHistory == null) { continue; } processorDescriptors.addAll(statusRepository.getProcessorMetricDescriptors()); // record the status history (last) to get the component details for use later final StatusHistoryDTO statusHistoryDto = createStatusHistoryDto(statusHistory); lastStatusHistory = statusHistoryDto; final NodeStatusHistoryDTO nodeHistory = new NodeStatusHistoryDTO(); nodeHistory.setStatusHistory(statusHistoryDto); nodeHistory.setNode(createNodeDTO(node)); nodeHistories.add(nodeHistory); // collect all of the snapshots to aggregate for (final StatusSnapshot snapshot : statusHistory.getStatusSnapshots()) { final Date normalizedDate = normalizeStatusSnapshotDate(snapshot.getTimestamp(), componentStatusSnapshotMillis); List<StatusSnapshot> snapshots = snapshotsToAggregate.get(normalizedDate); if (snapshots == null) { snapshots = new ArrayList<>(); snapshotsToAggregate.put(normalizedDate, snapshots); } snapshots.add(snapshot); } } // Aggregate the snapshots final List<StatusSnapshotDTO> aggregatedSnapshotDtos = aggregate(snapshotsToAggregate); // get the details for this component from the last status history final LinkedHashMap<String, String> clusterStatusHistoryDetails = new LinkedHashMap<>(); clusterStatusHistoryDetails.putAll(lastStatusHistory.getDetails()); final StatusHistoryDTO clusterStatusHistory = new StatusHistoryDTO(); clusterStatusHistory.setGenerated(new Date()); clusterStatusHistory.setFieldDescriptors(StatusHistoryUtil.createFieldDescriptorDtos(processorDescriptors)); clusterStatusHistory.setDetails(clusterStatusHistoryDetails); clusterStatusHistory.setStatusSnapshots(aggregatedSnapshotDtos); final ClusterStatusHistoryDTO history = new ClusterStatusHistoryDTO(); history.setGenerated(new Date()); history.setNodeStatusHistory(nodeHistories); history.setClusterStatusHistory(clusterStatusHistory); return history; } public StatusHistoryDTO createStatusHistoryDto(final StatusHistory statusHistory) { final StatusHistoryDTO dto = new StatusHistoryDTO(); dto.setDetails(new LinkedHashMap<>(statusHistory.getComponentDetails())); dto.setFieldDescriptors(StatusHistoryUtil.createFieldDescriptorDtos(statusHistory)); dto.setGenerated(statusHistory.getDateGenerated()); final List<StatusSnapshotDTO> statusSnapshots = new ArrayList<>(); for (final StatusSnapshot statusSnapshot : statusHistory.getStatusSnapshots()) { statusSnapshots.add(StatusHistoryUtil.createStatusSnapshotDto(statusSnapshot)); } dto.setStatusSnapshots(statusSnapshots); return dto; } public ClusterStatusHistoryDTO getConnectionStatusHistory(final String connectionId) { return getConnectionStatusHistory(connectionId, null, null, Integer.MAX_VALUE); } public ClusterStatusHistoryDTO getConnectionStatusHistory(final String connectionId, final Date startDate, final Date endDate, final int preferredDataPoints) { final List<NodeStatusHistoryDTO> nodeHistories = new ArrayList<>(); StatusHistoryDTO lastStatusHistory = null; final Set<MetricDescriptor<?>> connectionDescriptors = new LinkedHashSet<>(); final Map<Date, List<StatusSnapshot>> snapshotsToAggregate = new TreeMap<>(); for (final Node node : getRawNodes()) { final ComponentStatusRepository statusRepository = componentMetricsRepositoryMap.get(node.getNodeId()); if (statusRepository == null) { continue; } final StatusHistory statusHistory = statusRepository.getConnectionStatusHistory(connectionId, startDate, endDate, preferredDataPoints); if (statusHistory == null) { continue; } final StatusHistoryDTO statusHistoryDto = createStatusHistoryDto(statusHistory); // record the status history (last) to get the componet details for use later lastStatusHistory = statusHistoryDto; connectionDescriptors.addAll(statusRepository.getConnectionMetricDescriptors()); final NodeStatusHistoryDTO nodeHistory = new NodeStatusHistoryDTO(); nodeHistory.setStatusHistory(statusHistoryDto); nodeHistory.setNode(createNodeDTO(node)); nodeHistories.add(nodeHistory); // collect all of the snapshots to aggregate for (final StatusSnapshot snapshot : statusHistory.getStatusSnapshots()) { final Date normalizedDate = normalizeStatusSnapshotDate(snapshot.getTimestamp(), componentStatusSnapshotMillis); List<StatusSnapshot> snapshots = snapshotsToAggregate.get(normalizedDate); if (snapshots == null) { snapshots = new ArrayList<>(); snapshotsToAggregate.put(normalizedDate, snapshots); } snapshots.add(snapshot); } } // Aggregate the snapshots final List<StatusSnapshotDTO> aggregatedSnapshotDtos = aggregate(snapshotsToAggregate); // get the details for this component from the last status history final LinkedHashMap<String, String> clusterStatusHistoryDetails = new LinkedHashMap<>(); clusterStatusHistoryDetails.putAll(lastStatusHistory.getDetails()); final StatusHistoryDTO clusterStatusHistory = new StatusHistoryDTO(); clusterStatusHistory.setGenerated(new Date()); clusterStatusHistory .setFieldDescriptors(StatusHistoryUtil.createFieldDescriptorDtos(connectionDescriptors)); clusterStatusHistory.setDetails(clusterStatusHistoryDetails); clusterStatusHistory.setStatusSnapshots(aggregatedSnapshotDtos); final ClusterStatusHistoryDTO history = new ClusterStatusHistoryDTO(); history.setGenerated(new Date()); history.setNodeStatusHistory(nodeHistories); history.setClusterStatusHistory(clusterStatusHistory); return history; } public ClusterStatusHistoryDTO getProcessGroupStatusHistory(final String processGroupId) { return getProcessGroupStatusHistory(processGroupId, null, null, Integer.MAX_VALUE); } public ClusterStatusHistoryDTO getProcessGroupStatusHistory(final String processGroupId, final Date startDate, final Date endDate, final int preferredDataPoints) { final List<NodeStatusHistoryDTO> nodeHistories = new ArrayList<>(); StatusHistoryDTO lastStatusHistory = null; final Set<MetricDescriptor<?>> processGroupDescriptors = new LinkedHashSet<>(); final Map<Date, List<StatusSnapshot>> snapshotsToAggregate = new TreeMap<>(); for (final Node node : getRawNodes()) { final ComponentStatusRepository statusRepository = componentMetricsRepositoryMap.get(node.getNodeId()); if (statusRepository == null) { continue; } final StatusHistory statusHistory = statusRepository.getProcessGroupStatusHistory(processGroupId, startDate, endDate, preferredDataPoints); if (statusHistory == null) { continue; } final StatusHistoryDTO statusHistoryDto = createStatusHistoryDto(statusHistory); // record the status history (last) to get the componet details for use later lastStatusHistory = statusHistoryDto; processGroupDescriptors.addAll(statusRepository.getProcessGroupMetricDescriptors()); final NodeStatusHistoryDTO nodeHistory = new NodeStatusHistoryDTO(); nodeHistory.setStatusHistory(statusHistoryDto); nodeHistory.setNode(createNodeDTO(node)); nodeHistories.add(nodeHistory); // collect all of the snapshots to aggregate for (final StatusSnapshot snapshot : statusHistory.getStatusSnapshots()) { final Date normalizedDate = normalizeStatusSnapshotDate(snapshot.getTimestamp(), componentStatusSnapshotMillis); List<StatusSnapshot> snapshots = snapshotsToAggregate.get(normalizedDate); if (snapshots == null) { snapshots = new ArrayList<>(); snapshotsToAggregate.put(normalizedDate, snapshots); } snapshots.add(snapshot); } } // Aggregate the snapshots final List<StatusSnapshotDTO> aggregatedSnapshotDtos = aggregate(snapshotsToAggregate); // get the details for this component from the last status history final LinkedHashMap<String, String> clusterStatusHistoryDetails = new LinkedHashMap<>(); clusterStatusHistoryDetails.putAll(lastStatusHistory.getDetails()); final StatusHistoryDTO clusterStatusHistory = new StatusHistoryDTO(); clusterStatusHistory.setGenerated(new Date()); clusterStatusHistory.setDetails(clusterStatusHistoryDetails); clusterStatusHistory .setFieldDescriptors(StatusHistoryUtil.createFieldDescriptorDtos(processGroupDescriptors)); clusterStatusHistory.setStatusSnapshots(aggregatedSnapshotDtos); final ClusterStatusHistoryDTO history = new ClusterStatusHistoryDTO(); history.setGenerated(new Date()); history.setNodeStatusHistory(nodeHistories); history.setClusterStatusHistory(clusterStatusHistory); return history; } public ClusterStatusHistoryDTO getRemoteProcessGroupStatusHistory(final String remoteGroupId) { return getRemoteProcessGroupStatusHistory(remoteGroupId, null, null, Integer.MAX_VALUE); } public ClusterStatusHistoryDTO getRemoteProcessGroupStatusHistory(final String remoteGroupId, final Date startDate, final Date endDate, final int preferredDataPoints) { final List<NodeStatusHistoryDTO> nodeHistories = new ArrayList<>(); StatusHistoryDTO lastStatusHistory = null; final Set<MetricDescriptor<?>> remoteProcessGroupDescriptors = new LinkedHashSet<>(); final Map<Date, List<StatusSnapshot>> snapshotsToAggregate = new TreeMap<>(); for (final Node node : getRawNodes()) { final ComponentStatusRepository statusRepository = componentMetricsRepositoryMap.get(node.getNodeId()); if (statusRepository == null) { continue; } final StatusHistory statusHistory = statusRepository.getRemoteProcessGroupStatusHistory(remoteGroupId, startDate, endDate, preferredDataPoints); if (statusHistory == null) { continue; } final StatusHistoryDTO statusHistoryDto = createStatusHistoryDto(statusHistory); // record the status history (last) to get the componet details for use later lastStatusHistory = statusHistoryDto; remoteProcessGroupDescriptors.addAll(statusRepository.getRemoteProcessGroupMetricDescriptors()); final NodeStatusHistoryDTO nodeHistory = new NodeStatusHistoryDTO(); nodeHistory.setStatusHistory(statusHistoryDto); nodeHistory.setNode(createNodeDTO(node)); nodeHistories.add(nodeHistory); // collect all of the snapshots to aggregate for (final StatusSnapshot snapshot : statusHistory.getStatusSnapshots()) { final Date normalizedDate = normalizeStatusSnapshotDate(snapshot.getTimestamp(), componentStatusSnapshotMillis); List<StatusSnapshot> snapshots = snapshotsToAggregate.get(normalizedDate); if (snapshots == null) { snapshots = new ArrayList<>(); snapshotsToAggregate.put(normalizedDate, snapshots); } snapshots.add(snapshot); } } // Aggregate the snapshots final List<StatusSnapshotDTO> aggregatedSnapshotDtos = aggregate(snapshotsToAggregate); // get the details for this component from the last status history final LinkedHashMap<String, String> clusterStatusHistoryDetails = new LinkedHashMap<>(); clusterStatusHistoryDetails.putAll(lastStatusHistory.getDetails()); final StatusHistoryDTO clusterStatusHistory = new StatusHistoryDTO(); clusterStatusHistory.setGenerated(new Date()); clusterStatusHistory.setDetails(clusterStatusHistoryDetails); clusterStatusHistory .setFieldDescriptors(StatusHistoryUtil.createFieldDescriptorDtos(remoteProcessGroupDescriptors)); clusterStatusHistory.setStatusSnapshots(aggregatedSnapshotDtos); final ClusterStatusHistoryDTO history = new ClusterStatusHistoryDTO(); history.setGenerated(new Date()); history.setNodeStatusHistory(nodeHistories); history.setClusterStatusHistory(clusterStatusHistory); return history; } private static class ClusterManagerLock { private final Lock lock; private static final Logger logger = LoggerFactory.getLogger("cluster.lock"); private long lockTime; private final String name; public ClusterManagerLock(final Lock lock, final String name) { this.lock = lock; this.name = name; } @SuppressWarnings("unused") public boolean tryLock() { logger.trace("Trying to obtain Cluster Manager Lock: {}", name); final boolean success = lock.tryLock(); if (!success) { logger.trace("TryLock failed for Cluster Manager Lock: {}", name); return false; } logger.trace("TryLock successful"); return true; } public boolean tryLock(final long timeout, final TimeUnit timeUnit) { logger.trace("Trying to obtain Cluster Manager Lock {} with a timeout of {} {}", name, timeout, timeUnit); final boolean success; try { success = lock.tryLock(timeout, timeUnit); } catch (final InterruptedException ie) { return false; } if (!success) { logger.trace("TryLock failed for Cluster Manager Lock {} with a timeout of {} {}", name, timeout, timeUnit); return false; } logger.trace("TryLock successful"); return true; } public void lock() { logger.trace("Obtaining Cluster Manager Lock {}", name); lock.lock(); lockTime = System.nanoTime(); logger.trace("Obtained Cluster Manager Lock {}", name); } public void unlock(final String task) { logger.trace("Releasing Cluster Manager Lock {}", name); final long nanosLocked = System.nanoTime() - lockTime; lock.unlock(); logger.trace("Released Cluster Manager Lock {}", name); final long millisLocked = TimeUnit.MILLISECONDS.convert(nanosLocked, TimeUnit.NANOSECONDS); if (millisLocked > 100L) { logger.debug("Cluster Manager Lock {} held for {} milliseconds for task: {}", name, millisLocked, task); } } } @Override public Set<String> getControllerServiceIdentifiers(final Class<? extends ControllerService> serviceType) { return controllerServiceProvider.getControllerServiceIdentifiers(serviceType); } }