gobblin.cluster.GobblinClusterManager.java Source code

Java tutorial

Introduction

Here is the source code for gobblin.cluster.GobblinClusterManager.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package gobblin.cluster;

import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.net.URI;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.UUID;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.helix.ControllerChangeListener;
import org.apache.helix.Criteria;
import org.apache.helix.HelixDataAccessor;
import org.apache.helix.HelixManager;
import org.apache.helix.HelixManagerFactory;
import org.apache.helix.HelixProperty;
import org.apache.helix.InstanceType;
import org.apache.helix.LiveInstanceChangeListener;
import org.apache.helix.NotificationContext;
import org.apache.helix.messaging.handling.HelixTaskResult;
import org.apache.helix.messaging.handling.MessageHandler;
import org.apache.helix.messaging.handling.MessageHandlerFactory;
import org.apache.helix.model.LiveInstance;
import org.apache.helix.model.Message;
import org.apache.helix.task.TaskDriver;
import org.apache.helix.task.TaskUtil;
import org.apache.helix.task.WorkflowConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Optional;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.eventbus.EventBus;
import com.google.common.eventbus.Subscribe;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.Service;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigFactory;
import com.typesafe.config.ConfigValueFactory;

import gobblin.annotation.Alpha;
import gobblin.cluster.event.ClusterManagerShutdownRequest;
import gobblin.configuration.ConfigurationKeys;
import gobblin.metrics.Tag;
import gobblin.runtime.api.MutableJobCatalog;
import gobblin.runtime.app.ApplicationException;
import gobblin.runtime.app.ApplicationLauncher;
import gobblin.runtime.app.ServiceBasedAppLauncher;
import gobblin.scheduler.SchedulerService;
import gobblin.util.ConfigUtils;
import gobblin.util.JvmUtils;
import gobblin.util.logs.Log4jConfigurationHelper;
import gobblin.util.reflection.GobblinConstructorUtils;

/**
 * The central cluster manager for Gobblin Clusters.
 *
 * <p>
 *   This class runs the {@link GobblinHelixJobScheduler} for scheduling and running Gobblin jobs.
 *   This class serves as the Helix controller and it uses a {@link HelixManager} to work with Helix.
 * </p>
 *
 * <p>
 *   This class will initiates a graceful shutdown of the cluster in the following conditions:
 *
 *   <ul>
 *     <li>A shutdown request is received via a Helix message of subtype
 *     {@link HelixMessageSubTypes#APPLICATION_MASTER_SHUTDOWN}. Upon receiving such a message,
 *     it will call {@link #stop()} to initiate a graceful shutdown of the cluster</li>
 *     <li>The shutdown hook gets called. The shutdown hook will call {@link #stop()}, which will
 *     start a graceful shutdown of the cluster.</li>
 *   </ul>
 * </p>
 *
 * @author Yinan Li
 */
@Alpha
public class GobblinClusterManager implements ApplicationLauncher {

    private static final Logger LOGGER = LoggerFactory.getLogger(GobblinClusterManager.class);

    private HelixManager helixManager;

    private volatile boolean stopInProgress = false;

    protected ServiceBasedAppLauncher applicationLauncher;

    // An EventBus used for communications between services running in the ApplicationMaster
    protected final EventBus eventBus = new EventBus(GobblinClusterManager.class.getSimpleName());

    protected final Path appWorkDir;

    protected final FileSystem fs;

    protected final String applicationId;

    // thread used to keep process up for an idle controller
    private Thread idleProcessThread;

    // set to true to stop the idle process thread
    private volatile boolean stopIdleProcessThread = false;

    // flag to keep track of leader and avoid processing duplicate leadership change notifications
    private boolean isLeader = false;

    private final boolean isStandaloneMode;

    private MutableJobCatalog jobCatalog;

    private final String clusterName;
    private final Config config;

    public GobblinClusterManager(String clusterName, String applicationId, Config config,
            Optional<Path> appWorkDirOptional) throws Exception {
        this.clusterName = clusterName;
        this.config = config;

        this.isStandaloneMode = ConfigUtils.getBoolean(config,
                GobblinClusterConfigurationKeys.STANDALONE_CLUSTER_MODE_KEY,
                GobblinClusterConfigurationKeys.DEFAULT_STANDALONE_CLUSTER_MODE);

        this.applicationId = applicationId;

        initializeHelixManager();

        this.fs = buildFileSystem(config);
        this.appWorkDir = appWorkDirOptional.isPresent() ? appWorkDirOptional.get()
                : GobblinClusterUtils.getAppWorkDirPath(this.fs, clusterName, applicationId);

        initializeAppLauncherAndServices();
    }

    /**
     * Create the service based application launcher and other associated services
     * @throws Exception
     */
    private void initializeAppLauncherAndServices() throws Exception {
        // Done to preserve backwards compatibility with the previously hard-coded timeout of 5 minutes
        Properties properties = ConfigUtils.configToProperties(this.config);
        if (!properties.contains(ServiceBasedAppLauncher.APP_STOP_TIME_SECONDS)) {
            properties.setProperty(ServiceBasedAppLauncher.APP_STOP_TIME_SECONDS, Long.toString(300));
        }
        this.applicationLauncher = new ServiceBasedAppLauncher(properties, this.clusterName);

        // create a job catalog for keeping track of received jobs if a job config path is specified
        if (this.config.hasPath(GobblinClusterConfigurationKeys.GOBBLIN_CLUSTER_PREFIX
                + ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY)) {
            String jobCatalogClassName = ConfigUtils.getString(config,
                    GobblinClusterConfigurationKeys.JOB_CATALOG_KEY,
                    GobblinClusterConfigurationKeys.DEFAULT_JOB_CATALOG);

            this.jobCatalog = (MutableJobCatalog) GobblinConstructorUtils.invokeFirstConstructor(
                    Class.forName(jobCatalogClassName), ImmutableList.<Object>of(config.getConfig(
                            StringUtils.removeEnd(GobblinClusterConfigurationKeys.GOBBLIN_CLUSTER_PREFIX, "."))));
        } else {
            this.jobCatalog = null;
        }

        SchedulerService schedulerService = new SchedulerService(properties);
        this.applicationLauncher.addService(schedulerService);
        this.applicationLauncher.addService(buildGobblinHelixJobScheduler(config, this.appWorkDir,
                getMetadataTags(clusterName, applicationId), schedulerService));
        this.applicationLauncher.addService(buildJobConfigurationManager(config));
    }

    /**
     * Start any services required by the application launcher then start the application launcher
     */
    private void startAppLauncherAndServices() {
        // other services such as the job configuration manager have a dependency on the job catalog, so it has be be
        // started first
        if (this.jobCatalog instanceof Service) {
            ((Service) this.jobCatalog).startAsync().awaitRunning();
        }

        this.applicationLauncher.start();
    }

    /**
     * Stop the application launcher then any services that were started outside of the application launcher
     */
    private void stopAppLauncherAndServices() {
        try {
            this.applicationLauncher.stop();
        } catch (ApplicationException ae) {
            LOGGER.error("Error while stopping Gobblin Cluster application launcher", ae);
        }

        if (this.jobCatalog instanceof Service) {
            ((Service) this.jobCatalog).stopAsync().awaitTerminated();
        }
    }

    /**
     * Handle leadership change.
     * The applicationLauncher is only started on the leader.
     * The leader cleans up existing jobs before starting the applicationLauncher.
     * @param changeContext notification context
     */
    @VisibleForTesting
    void handleLeadershipChange(NotificationContext changeContext) {
        if (this.helixManager.isLeader()) {
            // can get multiple notifications on a leadership change, so only start the application launcher the first time
            // the notification is received
            LOGGER.info("Leader notification for {} isLeader {} HM.isLeader {}",
                    this.helixManager.getInstanceName(), isLeader, this.helixManager.isLeader());

            if (!isLeader) {
                LOGGER.info("New Helix Controller leader {}", this.helixManager.getInstanceName());

                // Clean up existing jobs
                TaskDriver taskDriver = new TaskDriver(this.helixManager);
                GobblinHelixTaskDriver gobblinHelixTaskDriver = new GobblinHelixTaskDriver(this.helixManager);
                Map<String, WorkflowConfig> workflows = taskDriver.getWorkflows();

                for (Map.Entry<String, WorkflowConfig> entry : workflows.entrySet()) {
                    String queueName = entry.getKey();
                    WorkflowConfig workflowConfig = entry.getValue();

                    for (String namespacedJobName : workflowConfig.getJobDag().getAllNodes()) {
                        String jobName = TaskUtil.getDenamespacedJobName(queueName, namespacedJobName);
                        LOGGER.info("job {} found for queue {} ", jobName, queueName);

                        // #HELIX-0.6.7-WORKAROUND
                        // working around 0.6.7 delete job issue for queues with IN_PROGRESS state
                        gobblinHelixTaskDriver.deleteJob(queueName, jobName);
                        LOGGER.info("deleted job {} from queue {}", jobName, queueName);
                    }
                }

                startAppLauncherAndServices();
                isLeader = true;
            }
        } else {
            // stop and reinitialize services since they are not restartable
            // this prepares them to start when this cluster manager becomes a leader
            if (isLeader) {
                isLeader = false;
                stopAppLauncherAndServices();
                try {
                    initializeAppLauncherAndServices();
                } catch (Exception e) {
                    throw new RuntimeException("Exception reinitializing app launcher services ", e);
                }
            }
        }
    }

    /**
     * Start the Gobblin Cluster Manager.
     */
    @Override
    public synchronized void start() {
        LOGGER.info("Starting the Gobblin Cluster Manager");

        this.eventBus.register(this);
        connectHelixManager();

        if (this.isStandaloneMode) {
            // standalone mode starts non-daemon threads later, so need to have this thread to keep process up
            this.idleProcessThread = new Thread(new Runnable() {
                @Override
                public void run() {
                    while (!GobblinClusterManager.this.stopInProgress
                            && !GobblinClusterManager.this.stopIdleProcessThread) {
                        try {
                            Thread.sleep(300);
                        } catch (InterruptedException e) {
                            Thread.currentThread().interrupt();
                            break;
                        }
                    }
                }
            });

            this.idleProcessThread.start();

            // Need this in case a kill is issued to the process so that the idle thread does not keep the process up
            // since GobblinClusterManager.stop() is not called this case.
            Runtime.getRuntime().addShutdownHook(new Thread() {
                @Override
                public void run() {
                    GobblinClusterManager.this.stopIdleProcessThread = true;
                }
            });
        } else {
            startAppLauncherAndServices();
        }
    }

    /**
     * Stop the Gobblin Cluster Manager.
     */
    @Override
    public synchronized void stop() {
        if (this.stopInProgress) {
            return;
        }

        this.stopInProgress = true;

        LOGGER.info("Stopping the Gobblin Cluster Manager");

        if (this.idleProcessThread != null) {
            try {
                this.idleProcessThread.join();
            } catch (InterruptedException ie) {
                Thread.currentThread().interrupt();
            }
        }

        // Send a shutdown request to all GobblinTaskRunners unless running in standalone mode.
        // In standalone mode a failing manager should not stop the whole cluster.
        if (!this.isStandaloneMode) {
            sendShutdownRequest();
        }

        stopAppLauncherAndServices();

        disconnectHelixManager();
    }

    /**
     * Get additional {@link Tag}s required for any type of reporting.
     */
    private List<? extends Tag<?>> getMetadataTags(String applicationName, String applicationId) {
        return Tag.fromMap(new ImmutableMap.Builder<String, Object>()
                .put(GobblinClusterMetricTagNames.APPLICATION_NAME, applicationName)
                .put(GobblinClusterMetricTagNames.APPLICATION_ID, applicationId).build());
    }

    /**
     * Build the {@link HelixManager} for the Application Master.
     */
    private HelixManager buildHelixManager(Config config, String zkConnectionString) {
        String helixInstanceName = ConfigUtils.getString(config,
                GobblinClusterConfigurationKeys.HELIX_INSTANCE_NAME_KEY,
                GobblinClusterManager.class.getSimpleName());
        return HelixManagerFactory.getZKHelixManager(
                config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY), helixInstanceName,
                InstanceType.CONTROLLER, zkConnectionString);
    }

    /**
     * Build the {@link FileSystem} for the Application Master.
     */
    private FileSystem buildFileSystem(Config config) throws IOException {
        return config.hasPath(ConfigurationKeys.FS_URI_KEY)
                ? FileSystem.get(URI.create(config.getString(ConfigurationKeys.FS_URI_KEY)), new Configuration())
                : FileSystem.get(new Configuration());
    }

    /**
     * Build the {@link GobblinHelixJobScheduler} for the Application Master.
     */
    private GobblinHelixJobScheduler buildGobblinHelixJobScheduler(Config config, Path appWorkDir,
            List<? extends Tag<?>> metadataTags, SchedulerService schedulerService) throws Exception {
        Properties properties = ConfigUtils.configToProperties(config);
        return new GobblinHelixJobScheduler(properties, this.helixManager, this.eventBus, appWorkDir, metadataTags,
                schedulerService, this.jobCatalog);
    }

    /**
     * Build the {@link JobConfigurationManager} for the Application Master.
     */
    private JobConfigurationManager buildJobConfigurationManager(Config config) {
        return create(config);
    }

    private JobConfigurationManager create(Config config) {
        try {
            if (config.hasPath(GobblinClusterConfigurationKeys.JOB_CONFIGURATION_MANAGER_KEY)) {
                return (JobConfigurationManager) GobblinConstructorUtils.invokeFirstConstructor(
                        Class.forName(
                                config.getString(GobblinClusterConfigurationKeys.JOB_CONFIGURATION_MANAGER_KEY)),
                        ImmutableList.<Object>of(this.eventBus, config, this.jobCatalog),
                        ImmutableList.<Object>of(this.eventBus, config));
            } else {
                return new JobConfigurationManager(this.eventBus, config);
            }
        } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException
                | ClassNotFoundException e) {
            throw new RuntimeException(e);
        }
    }

    @SuppressWarnings("unused")
    @Subscribe
    public void handleApplicationMasterShutdownRequest(ClusterManagerShutdownRequest shutdownRequest) {
        stop();
    }

    @VisibleForTesting
    EventBus getEventBus() {
        return this.eventBus;
    }

    @VisibleForTesting
    void connectHelixManager() {
        try {
            this.isLeader = false;
            this.helixManager.connect();
            this.helixManager.addLiveInstanceChangeListener(new GobblinLiveInstanceChangeListener());
            this.helixManager.getMessagingService().registerMessageHandlerFactory(
                    GobblinHelixConstants.SHUTDOWN_MESSAGE_TYPE, new ControllerShutdownMessageHandlerFactory());
            this.helixManager.getMessagingService().registerMessageHandlerFactory(
                    Message.MessageType.USER_DEFINE_MSG.toString(), getUserDefinedMessageHandlerFactory());

            // standalone mode listens for controller change
            if (this.isStandaloneMode) {
                // Subscribe to leadership changes
                this.helixManager.addControllerListener(new ControllerChangeListener() {
                    @Override
                    public void onControllerChange(NotificationContext changeContext) {
                        handleLeadershipChange(changeContext);
                    }
                });
            }
        } catch (Exception e) {
            LOGGER.error("HelixManager failed to connect", e);
            throw Throwables.propagate(e);
        }
    }

    /**
     * Creates and returns a {@link MessageHandlerFactory} for handling of Helix
     * {@link org.apache.helix.model.Message.MessageType#USER_DEFINE_MSG}s.
     *
     * @returns a {@link MessageHandlerFactory}.
     */
    protected MessageHandlerFactory getUserDefinedMessageHandlerFactory() {
        return new ControllerUserDefinedMessageHandlerFactory();
    }

    @VisibleForTesting
    void disconnectHelixManager() {
        if (isHelixManagerConnected()) {
            this.helixManager.disconnect();
        }
    }

    @VisibleForTesting
    boolean isHelixManagerConnected() {
        return this.helixManager.isConnected();
    }

    @VisibleForTesting
    void initializeHelixManager() {
        String zkConnectionString = this.config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY);
        LOGGER.info("Using ZooKeeper connection string: " + zkConnectionString);

        // This will create and register a Helix controller in ZooKeeper
        this.helixManager = buildHelixManager(this.config, zkConnectionString);
    }

    @VisibleForTesting
    void sendShutdownRequest() {
        Criteria criteria = new Criteria();
        criteria.setInstanceName("%");
        criteria.setResource("%");
        criteria.setPartition("%");
        criteria.setPartitionState("%");
        criteria.setRecipientInstanceType(InstanceType.PARTICIPANT);
        // #HELIX-0.6.7-WORKAROUND
        // Add this back when messaging to instances is ported to 0.6 branch
        //criteria.setDataSource(Criteria.DataSource.LIVEINSTANCES);
        criteria.setSessionSpecific(true);

        Message shutdownRequest = new Message(GobblinHelixConstants.SHUTDOWN_MESSAGE_TYPE,
                HelixMessageSubTypes.WORK_UNIT_RUNNER_SHUTDOWN.toString().toLowerCase()
                        + UUID.randomUUID().toString());
        shutdownRequest.setMsgSubType(HelixMessageSubTypes.WORK_UNIT_RUNNER_SHUTDOWN.toString());
        shutdownRequest.setMsgState(Message.MessageState.NEW);

        // Wait for 5 minutes
        final int timeout = 300000;

        // #HELIX-0.6.7-WORKAROUND
        // Temporarily bypass the default messaging service to allow upgrade to 0.6.7 which is missing support
        // for messaging to instances
        //int messagesSent = this.helixManager.getMessagingService().send(criteria, shutdownRequest,
        //    new NoopReplyHandler(), timeout);
        GobblinHelixMessagingService messagingService = new GobblinHelixMessagingService(this.helixManager);

        int messagesSent = messagingService.send(criteria, shutdownRequest, new NoopReplyHandler(), timeout);
        if (messagesSent == 0) {
            LOGGER.error(String.format("Failed to send the %s message to the participants",
                    shutdownRequest.getMsgSubType()));
        }
    }

    @Override
    public void close() throws IOException {
        this.applicationLauncher.close();
    }

    /**
     * A custom implementation of {@link LiveInstanceChangeListener}.
     */
    private static class GobblinLiveInstanceChangeListener implements LiveInstanceChangeListener {

        @Override
        public void onLiveInstanceChange(List<LiveInstance> liveInstances, NotificationContext changeContext) {
            for (LiveInstance liveInstance : liveInstances) {
                LOGGER.info("Live Helix participant instance: " + liveInstance.getInstanceName());
            }
        }
    }

    /**
     * A custom {@link MessageHandlerFactory} for {@link MessageHandler}s that handle messages of type
     * "SHUTDOWN" for shutting down the controller.
     */
    private class ControllerShutdownMessageHandlerFactory implements MessageHandlerFactory {

        @Override
        public MessageHandler createHandler(Message message, NotificationContext context) {
            return new ControllerShutdownMessageHandler(message, context);
        }

        @Override
        public String getMessageType() {
            return GobblinHelixConstants.SHUTDOWN_MESSAGE_TYPE;
        }

        public List<String> getMessageTypes() {
            return Collections.singletonList(getMessageType());
        }

        @Override
        public void reset() {

        }

        /**
         * A custom {@link MessageHandler} for handling messages of sub type
         * {@link HelixMessageSubTypes#APPLICATION_MASTER_SHUTDOWN}.
         */
        private class ControllerShutdownMessageHandler extends MessageHandler {

            public ControllerShutdownMessageHandler(Message message, NotificationContext context) {
                super(message, context);
            }

            @Override
            public HelixTaskResult handleMessage() throws InterruptedException {
                String messageSubType = this._message.getMsgSubType();
                Preconditions.checkArgument(
                        messageSubType
                                .equalsIgnoreCase(HelixMessageSubTypes.APPLICATION_MASTER_SHUTDOWN.toString()),
                        String.format("Unknown %s message subtype: %s", GobblinHelixConstants.SHUTDOWN_MESSAGE_TYPE,
                                messageSubType));

                HelixTaskResult result = new HelixTaskResult();

                if (stopInProgress) {
                    result.setSuccess(true);
                    return result;
                }

                LOGGER.info("Handling message " + HelixMessageSubTypes.APPLICATION_MASTER_SHUTDOWN.toString());

                ScheduledExecutorService shutdownMessageHandlingCompletionWatcher = MoreExecutors
                        .getExitingScheduledExecutorService(new ScheduledThreadPoolExecutor(1));

                // Schedule the task for watching on the removal of the shutdown message, which indicates that
                // the message has been successfully processed and it's safe to disconnect the HelixManager.
                // This is a hacky way of watching for the completion of processing the shutdown message and
                // should be replaced by a fix to https://issues.apache.org/jira/browse/HELIX-611.
                shutdownMessageHandlingCompletionWatcher.scheduleAtFixedRate(new Runnable() {
                    @Override
                    public void run() {
                        HelixManager helixManager = _notificationContext.getManager();
                        HelixDataAccessor helixDataAccessor = helixManager.getHelixDataAccessor();

                        HelixProperty helixProperty = helixDataAccessor.getProperty(
                                _message.getKey(helixDataAccessor.keyBuilder(), helixManager.getInstanceName()));
                        // The absence of the shutdown message indicates it has been removed
                        if (helixProperty == null) {
                            eventBus.post(new ClusterManagerShutdownRequest());
                        }
                    }
                }, 0, 1, TimeUnit.SECONDS);

                result.setSuccess(true);
                return result;
            }

            @Override
            public void onError(Exception e, ErrorCode code, ErrorType type) {
                LOGGER.error(String.format(
                        "Failed to handle message with exception %s, error code %s, error type %s", e, code, type));
            }
        }
    }

    /**
     * A custom {@link MessageHandlerFactory} for {@link ControllerUserDefinedMessageHandler}s that
     * handle messages of type {@link org.apache.helix.model.Message.MessageType#USER_DEFINE_MSG}.
     */
    private static class ControllerUserDefinedMessageHandlerFactory implements MessageHandlerFactory {

        @Override
        public MessageHandler createHandler(Message message, NotificationContext context) {
            return new ControllerUserDefinedMessageHandler(message, context);
        }

        @Override
        public String getMessageType() {
            return Message.MessageType.USER_DEFINE_MSG.toString();
        }

        public List<String> getMessageTypes() {
            return Collections.singletonList(getMessageType());
        }

        @Override
        public void reset() {

        }

        /**
         * A custom {@link MessageHandler} for handling user-defined messages to the controller.
         *
         * <p>
         *   Currently does not handle any user-defined messages. If this class is passed a custom message, it will simply
         *   print out a warning and return successfully. Sub-classes of {@link GobblinClusterManager} should override
         *   {@link #getUserDefinedMessageHandlerFactory}.
         * </p>
         */
        private static class ControllerUserDefinedMessageHandler extends MessageHandler {

            public ControllerUserDefinedMessageHandler(Message message, NotificationContext context) {
                super(message, context);
            }

            @Override
            public HelixTaskResult handleMessage() throws InterruptedException {
                LOGGER.warn(String.format("No handling setup for %s message of subtype: %s",
                        Message.MessageType.USER_DEFINE_MSG.toString(), this._message.getMsgSubType()));

                HelixTaskResult helixTaskResult = new HelixTaskResult();
                helixTaskResult.setSuccess(true);
                return helixTaskResult;
            }

            @Override
            public void onError(Exception e, ErrorCode code, ErrorType type) {
                LOGGER.error(String.format(
                        "Failed to handle message with exception %s, error code %s, error type %s", e, code, type));
            }
        }
    }

    /**
     * TODO for now the cluster id is hardcoded to 1 both here and in the {@link GobblinTaskRunner}. In the future, the
     * cluster id should be created by the {@link GobblinClusterManager} and passed to each {@link GobblinTaskRunner} via
     * Helix (at least that would be the easiest approach, there are certainly others ways to do it).
     */
    private static String getApplicationId() {
        return "1";
    }

    private static Options buildOptions() {
        Options options = new Options();
        options.addOption("a", GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME, true,
                "Gobblin application name");
        options.addOption("s", GobblinClusterConfigurationKeys.STANDALONE_CLUSTER_MODE, true,
                "Standalone cluster mode");
        options.addOption("i", GobblinClusterConfigurationKeys.HELIX_INSTANCE_NAME_OPTION_NAME, true,
                "Helix instance name");
        return options;
    }

    private static void printUsage(Options options) {
        HelpFormatter formatter = new HelpFormatter();
        formatter.printHelp(GobblinClusterManager.class.getSimpleName(), options);
    }

    public static void main(String[] args) throws Exception {
        Options options = buildOptions();
        try {
            CommandLine cmd = new DefaultParser().parse(options, args);
            if (!cmd.hasOption(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME)) {
                printUsage(options);
                System.exit(1);
            }

            boolean isStandaloneClusterManager = false;
            if (cmd.hasOption(GobblinClusterConfigurationKeys.STANDALONE_CLUSTER_MODE)) {
                isStandaloneClusterManager = Boolean.parseBoolean(
                        cmd.getOptionValue(GobblinClusterConfigurationKeys.STANDALONE_CLUSTER_MODE, "false"));
            }

            Log4jConfigurationHelper.updateLog4jConfiguration(GobblinClusterManager.class,
                    GobblinClusterConfigurationKeys.GOBBLIN_CLUSTER_LOG4J_CONFIGURATION_FILE,
                    GobblinClusterConfigurationKeys.GOBBLIN_CLUSTER_LOG4J_CONFIGURATION_FILE);

            LOGGER.info(JvmUtils.getJvmInputArguments());
            Config config = ConfigFactory.load();

            if (cmd.hasOption(GobblinClusterConfigurationKeys.HELIX_INSTANCE_NAME_OPTION_NAME)) {
                config = config.withValue(GobblinClusterConfigurationKeys.HELIX_INSTANCE_NAME_KEY,
                        ConfigValueFactory.fromAnyRef(cmd
                                .getOptionValue(GobblinClusterConfigurationKeys.HELIX_INSTANCE_NAME_OPTION_NAME)));
            }

            if (isStandaloneClusterManager) {
                config = config.withValue(GobblinClusterConfigurationKeys.STANDALONE_CLUSTER_MODE_KEY,
                        ConfigValueFactory.fromAnyRef(true));
            }

            try (GobblinClusterManager gobblinClusterManager = new GobblinClusterManager(
                    cmd.getOptionValue(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME),
                    getApplicationId(), config, Optional.<Path>absent())) {

                // In AWS / Yarn mode, the cluster Launcher takes care of setting up Helix cluster
                /// .. but for Standalone mode, we go via this main() method, so setup the cluster here
                if (isStandaloneClusterManager) {
                    // Create Helix cluster and connect to it
                    String zkConnectionString = config
                            .getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY);
                    String helixClusterName = config
                            .getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY);
                    HelixUtils.createGobblinHelixCluster(zkConnectionString, helixClusterName, false);
                    LOGGER.info("Created Helix cluster " + helixClusterName);
                }

                gobblinClusterManager.start();
            }
        } catch (ParseException pe) {
            printUsage(options);
            System.exit(1);
        }
    }
}