Java tutorial
/** * Copyright 2014 Otto (GmbH & Co KG) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.ottogroup.bi.asap.pipeline; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import com.ottogroup.bi.asap.component.Component; import com.ottogroup.bi.asap.component.emitter.Emitter; import com.ottogroup.bi.asap.component.emitter.executor.EmitterExecutor; import com.ottogroup.bi.asap.component.operator.Operator; import com.ottogroup.bi.asap.component.operator.executor.OperatorExecutor; import com.ottogroup.bi.asap.component.source.Source; import com.ottogroup.bi.asap.component.source.executor.SourceExecutor; import com.ottogroup.bi.asap.exception.ComponentAlreadySubmittedException; import com.ottogroup.bi.asap.exception.IllegalComponentSubscriptionException; import com.ottogroup.bi.asap.exception.RequiredInputMissingException; import com.ottogroup.bi.asap.exception.UnknownComponentException; import com.ottogroup.bi.asap.mailbox.Mailbox; /** * The micro pipeline provides the core runtime environment for {@link Source}, {@link Operator} and {@link Emitter} instances * which are encapsulated in {@link SourceExecutor}, {@link OperatorExecutor} or {@link EmitterExecutor} instances. Each * executor is assigned to its own {@link Thread} which is operated in an {@link ExecutorService} that is either externally provided * or exclusively created and maintained by the micro pipeline instance. * @author mnxfst * @since Dec 15, 2014 */ public class MicroPipeline { /** our faithful logging service ;-) */ private static final Logger logger = Logger.getLogger(MicroPipeline.class); /** unique identifier used for referencing the micro pipeline */ private final String id; /** optional description */ private final String description; /** runtime environment for managed executors */ private final ExecutorService executorService; /** was the executor service externally provided or internally created - required during shutdown */ private final boolean externalExecutorService; /** source executors managed by this micro pipeline */ private final Map<String, SourceExecutor> sourceExecutors = new HashMap<>(); /** operator executors managed by this micro pipeline */ private final Map<String, OperatorExecutor> operatorExecutors = new HashMap<>(); /** emitter executors managed by this micro pipeline */ private final Map<String, EmitterExecutor> emitterExecutors = new HashMap<>(); /** * Initializes the pipeline instance using the provided input. The newly created micro pipeline will use the * externally provided {@link ExecutorService} as runtime environment for all {@link SourceExecutor}, {@link OperatorExecutor} and {@link EmitterExecutor} instances. * @param id unique identifier used to reference the micro pipeline (required) * @param description pipeline description (optional) * @param executorService externally provided {@link ExecutorService executor service} (required) * @throws RequiredInputMissingException thrown in case any of the required parameters show no value */ public MicroPipeline(final String id, final String description, final ExecutorService executorService) throws RequiredInputMissingException { /////////////////////////////////////////////////////////////////// // validate input if (StringUtils.isBlank(id)) throw new RequiredInputMissingException("Missing required input for 'id'"); if (executorService == null) throw new RequiredInputMissingException("Missing required executor service"); // /////////////////////////////////////////////////////////////////// this.id = id; this.description = description; this.executorService = executorService; this.externalExecutorService = true; if (logger.isDebugEnabled()) logger.debug("micro pipeline instantiated [id=" + id + ", description=" + description + ", executorService=external]"); } /** * Initializes the pipeline instance using the provided input. The newly created micro pipeline will use an {@link ExecutorService} * created through {@link Executors#newCachedThreadPool()} as runtime environment for all {@link SourceExecutor}, {@link OperatorExecutor} * and {@link EmitterExecutor} instances. * @param id unique identifier used to reference the micro pipeline (required) * @param description pipeline description (optional) * @throws RequiredInputMissingException thrown in case any of the required parameters show no value */ public MicroPipeline(final String id, final String description) throws RequiredInputMissingException { /////////////////////////////////////////////////////////////////// // validate input if (StringUtils.isBlank(id)) throw new RequiredInputMissingException("Missing required input for 'id'"); // /////////////////////////////////////////////////////////////////// this.id = id; this.description = description; this.executorService = Executors.newCachedThreadPool(); this.externalExecutorService = false; if (logger.isDebugEnabled()) logger.debug("micro pipeline instantiated [id=" + id + ", description=" + description + ", executorService=internal]"); } /** * Initializes the pipeline instance using the provided input. The newly created micro pipeline will use an {@link ExecutorService} * created through {@link Executors#newFixedThreadPool(int)} as runtime environment for all {@link SourceExecutor}, {@link OperatorExecutor} * and {@link EmitterExecutor} instances. * @param id unique identifier used to reference the micro pipeline (required) * @param description pipeline description (optional) * @param threadPoolSize size to apply when creating the runtime environment ({@link Executors#newFixedThreadPool(int)}) (required) * @throws RequiredInputMissingException thrown in case any of the required parameters show no value */ public MicroPipeline(final String id, final String description, final int threadPoolSize) throws RequiredInputMissingException { /////////////////////////////////////////////////////////////////// // validate input if (StringUtils.isBlank(id)) throw new RequiredInputMissingException("Missing required input for 'id'"); if (threadPoolSize < 1) throw new RequiredInputMissingException( "Invalid thread pool size: " + threadPoolSize + ". Value must be greater than zero"); // /////////////////////////////////////////////////////////////////// this.id = id; this.description = description; this.executorService = Executors.newFixedThreadPool(threadPoolSize); this.externalExecutorService = false; if (logger.isDebugEnabled()) logger.debug("micro pipeline instantiated [id=" + id + ", description=" + description + ", executorService=internal]"); } /** * Submits the provided {@link SourceExecutor} to the pipeline instance * @param source * @throws RequiredInputMissingException * @throws ComponentAlreadySubmittedException */ public void submitSourceExecutor(final SourceExecutor sourceExecutor) throws RequiredInputMissingException, ComponentAlreadySubmittedException { /////////////////////////////////////////////////////////////////// // validate input if (sourceExecutor == null) throw new RequiredInputMissingException("Missing required input for 'sourceExecutor'"); if (StringUtils.isBlank(sourceExecutor.getSourceId())) throw new RequiredInputMissingException("Missing required input for source identifier"); // /////////////////////////////////////////////////////////////////// // normalize identifier to lower case String sid = StringUtils.lowerCase(StringUtils.trim(sourceExecutor.getSourceId())); if (this.sourceExecutors.containsKey(sid)) throw new ComponentAlreadySubmittedException( "Component '" + sourceExecutor.getSourceId() + "' already submitted to pipeline '" + id + "'"); // add source executor to internal map and submit it to executor service this.sourceExecutors.put(sid, sourceExecutor); this.executorService.submit(sourceExecutor); if (logger.isDebugEnabled()) logger.debug("source executor submitted [pid=" + id + ", sid=" + sid + ", mb=source has no mailbox]"); } /** * Submits the provided {@link OperatorExecutor} to the pipeline instance * @param operatorExecutor * @throws RequiredInputMissingException * @throws ComponentAlreadySubmittedException */ public void submitOperatorExecutor(final OperatorExecutor operatorExecutor) throws RequiredInputMissingException, ComponentAlreadySubmittedException { /////////////////////////////////////////////////////////////////// // validate input if (operatorExecutor == null) throw new RequiredInputMissingException("Missing required input for 'operatorExecutor'"); if (StringUtils.isBlank(operatorExecutor.getOperatorId())) throw new RequiredInputMissingException("Missing required input for operator identifier"); // /////////////////////////////////////////////////////////////////// // normalize identifier to lower case String oid = StringUtils.lowerCase(StringUtils.trim(operatorExecutor.getOperatorId())); if (this.operatorExecutors.containsKey(oid)) throw new ComponentAlreadySubmittedException("Component '" + operatorExecutor.getOperatorId() + "' already submitted to pipeline '" + id + "'"); // add operator executor to internal map and submit it to executor service this.operatorExecutors.put(oid, operatorExecutor); this.executorService.submit(operatorExecutor); if (logger.isDebugEnabled()) logger.debug("operator executor submitted [pid=" + id + ", oid=" + oid + ", mb=" + operatorExecutor.getMailbox() + "]"); } /** * Submits the provided {@link EmitterExecutor} to the pipeline instance * @param emitterExecutor * @throws RequiredInputMissingException * @throws ComponentAlreadySubmittedException */ public void submitEmitterExecutor(final EmitterExecutor emitterExecutor) throws RequiredInputMissingException, ComponentAlreadySubmittedException { /////////////////////////////////////////////////////////////////// // validate input if (emitterExecutor == null) throw new RequiredInputMissingException("Missing required input for 'emitterExecutor'"); if (StringUtils.isBlank(emitterExecutor.getEmitterId())) throw new RequiredInputMissingException("Missing required input for emitter identifier"); // /////////////////////////////////////////////////////////////////// // normalize identifier to lower case String eid = StringUtils.lowerCase(StringUtils.trim(emitterExecutor.getEmitterId())); if (this.emitterExecutors.containsKey(eid)) throw new ComponentAlreadySubmittedException("Component '" + emitterExecutor.getEmitterId() + "' already submitted to pipeline '" + id + "'"); // add emitter executor to internal map and submit it to executor service this.emitterExecutors.put(eid, emitterExecutor); this.executorService.submit(emitterExecutor); if (logger.isDebugEnabled()) logger.debug("emitter executor submitted [pid=" + id + ", eid=" + eid + ", mb=" + emitterExecutor.getMailbox() + "]"); } /** * Subscribes the {@link Component} referenced by the <i>subscriberId</i> to the {@link Component} referenced by the <i>publisherId</i> * @param subscriberId * @param publisherId * @throws RequiredInputMissingException * @throws UnknownComponentException * @throws IllegalComponentSubscriptionException thrown in case a component tries to subscribe itself to an {@link Emitter} or a {@link Source} tries to subscribe itself somewhere else */ public void subscribe(final String subscriberId, final String publisherId) throws RequiredInputMissingException, UnknownComponentException, IllegalComponentSubscriptionException { /////////////////////////////////////////////////////////////////// // validate input if (StringUtils.isBlank(subscriberId)) throw new RequiredInputMissingException("Missing required input for 'subscriberId'"); if (StringUtils.isBlank(publisherId)) throw new RequiredInputMissingException("Missing required input for 'publisherId'"); if (StringUtils.equalsIgnoreCase(subscriberId, publisherId)) throw new IllegalComponentSubscriptionException("Self-subscriptions are not permitted [subscriberId='" + subscriberId + "', publisherId='" + publisherId + "']"); // /////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////// // ensure that the subscriber id does not reference a source component, // ensure that the publisher id does not reference an emitter component and // ensure that subscriber id and publisher id are different String sid = StringUtils.lowerCase(StringUtils.trim(subscriberId)); String pid = StringUtils.lowerCase(StringUtils.trim(publisherId)); if (this.sourceExecutors.containsKey(sid)) throw new IllegalComponentSubscriptionException("Source component '" + subscriberId + "' is not allowed to subscribe itself to another component"); if (this.emitterExecutors.containsKey(pid)) throw new IllegalComponentSubscriptionException("Emitter component '" + publisherId + "' is not allowed to serve as publisher for another component"); // execute subscription Mailbox subscriberMailbox = null; if (this.operatorExecutors.containsKey(sid)) { OperatorExecutor subscriber = this.operatorExecutors.get(sid); if (subscriber == null) throw new UnknownComponentException( "Unknown component referenced by subscriberId '" + subscriberId + "'"); subscriberMailbox = subscriber.getMailbox(); } else if (this.emitterExecutors.containsKey(sid)) { EmitterExecutor subscriber = this.emitterExecutors.get(sid); if (subscriber == null) throw new UnknownComponentException( "Unknown component referenced by subscriberId '" + subscriberId + "'"); subscriberMailbox = subscriber.getMailbox(); } else { throw new UnknownComponentException( "Unknown component referenced by subscriberId '" + subscriberId + "'"); } if (this.operatorExecutors.containsKey(pid)) { OperatorExecutor publisher = this.operatorExecutors.get(pid); if (publisher == null) throw new UnknownComponentException( "Unknown component referenced by publisherId '" + publisherId + "'"); publisher.subscribe(sid, subscriberMailbox); } else if (this.sourceExecutors.containsKey(pid)) { SourceExecutor publisher = this.sourceExecutors.get(pid); if (publisher == null) throw new UnknownComponentException( "Unknown component referenced by publisherId '" + publisherId + "'"); publisher.subscribe(sid, subscriberMailbox); } else { throw new UnknownComponentException( "Unknown component referenced by publisherId '" + publisherId + "'"); } // /////////////////////////////////////////////////////////////////// } /** * Unsubscribes the referenced subscriber from the referenced publisher * @param subscriberId * @param publisherId * @throws RequiredInputMissingException */ public void unsubscribe(final String subscriberId, final String publisherId) throws RequiredInputMissingException, UnknownComponentException { /////////////////////////////////////////////////////////////////// // validate input if (StringUtils.isBlank(subscriberId)) throw new RequiredInputMissingException("Missing required input for 'subscriberId'"); if (StringUtils.isBlank(publisherId)) throw new RequiredInputMissingException("Missing required input for 'publisherId'"); // /////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////// // String pid = StringUtils.lowerCase(StringUtils.trim(publisherId)); if (this.sourceExecutors.containsKey(pid)) { SourceExecutor publisher = this.sourceExecutors.get(pid); if (publisher == null) throw new UnknownComponentException( "Unknown component referenced by publisherId '" + publisherId + "'"); publisher.unsubscribe(subscriberId); } else if (this.operatorExecutors.containsKey(pid)) { OperatorExecutor publisher = this.operatorExecutors.get(pid); if (publisher == null) throw new UnknownComponentException( "Unknown component referenced by publisherId '" + publisherId + "'"); publisher.unsubscribe(subscriberId); } else { throw new UnknownComponentException( "Unknown component referenced by publisherId '" + publisherId + "'"); } // /////////////////////////////////////////////////////////////////// if (logger.isDebugEnabled()) logger.debug("unsubscription [subscriberId=" + subscriberId + ", publisherId=" + publisherId + "]"); } /** * Shuts down the pipeline */ public void shutdown() { /////////////////////////////////////////////////////////////////////////////// // shut down sources, operators and emitters before shutting down the executor if (this.sourceExecutors != null && !this.sourceExecutors.isEmpty()) { for (SourceExecutor se : this.sourceExecutors.values()) { try { se.shutdown(); } catch (Exception e) { logger.error("Failed to shut down source executor '" + (se != null ? se.getSourceId() : null) + "'. Error: " + e.getMessage()); } } } if (this.operatorExecutors != null && !this.operatorExecutors.isEmpty()) { for (OperatorExecutor oe : this.operatorExecutors.values()) { try { oe.shutdown(); } catch (Exception e) { logger.error("Failed to shut down operator executor '" + (oe != null ? oe.getOperatorId() : null) + "'. Error: " + e.getMessage()); } } } if (this.emitterExecutors != null && !this.emitterExecutors.isEmpty()) { for (EmitterExecutor ee : this.emitterExecutors.values()) { try { ee.shutdown(); } catch (Exception e) { logger.error("Failed to shut down emitter executor '" + (ee != null ? ee.getEmitterId() : null) + "'. Error: " + e.getMessage()); } } } // /////////////////////////////////////////////////////////////////////////////// if (!externalExecutorService && this.executorService != null) this.executorService.shutdownNow(); } /** * Returns the unique identifier assigned to this micro pipeline * @return */ public String getId() { return this.id; } /** * Returns the micro pipeline description * @return */ public String getDescription() { return description; } }