info.pancancer.arch3.containerProvisioner.ContainerProvisionerThreads.java Source code

Java tutorial

Introduction

Here is the source code for info.pancancer.arch3.containerProvisioner.ContainerProvisionerThreads.java

Source

/*
 *     Consonance - workflow software for multiple clouds
 *     Copyright (C) 2016 OICR
 *
 *     This program is free software: you can redistribute it and/or modify
 *     it under the terms of the GNU General Public License as published by
 *     the Free Software Foundation, either version 3 of the License, or
 *     (at your option) any later version.
 *
 *     This program is distributed in the hope that it will be useful,
 *     but WITHOUT ANY WARRANTY; without even the implied warranty of
 *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *     GNU General Public License for more details.
 *
 *     You should have received a copy of the GNU General Public License
 *     along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package info.pancancer.arch3.containerProvisioner;

import com.google.gson.Gson;
import com.rabbitmq.client.Channel;
import com.rabbitmq.client.ConsumerCancelledException;
import com.rabbitmq.client.QueueingConsumer;
import com.rabbitmq.client.ShutdownSignalException;
import info.pancancer.arch3.Base;
import info.pancancer.arch3.beans.JobState;
import info.pancancer.arch3.beans.Provision;
import info.pancancer.arch3.beans.ProvisionState;
import info.pancancer.arch3.beans.Status;
import info.pancancer.arch3.beans.StatusState;
import info.pancancer.arch3.persistence.PostgreSQL;
import info.pancancer.arch3.utils.Constants;
import info.pancancer.arch3.utils.Utilities;
import info.pancancer.arch3.worker.WorkerRunnable;
import io.cloudbindle.youxia.deployer.Deployer;
import io.cloudbindle.youxia.reaper.Reaper;
import java.io.BufferedWriter;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeoutException;
import joptsimple.OptionSpecBuilder;
import org.apache.commons.configuration.HierarchicalINIConfiguration;
import org.apache.commons.exec.CommandLine;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Created by boconnor on 15-04-18.
 */
public class ContainerProvisionerThreads extends Base {

    private static final int DEFAULT_THREADS = 3;
    private static final int MINUTE_IN_MILLISECONDS = 60 * 1000;
    private static final int TWO_MINUTE_IN_MILLISECONDS = 2 * 60 * 1000;

    private final OptionSpecBuilder testSpec;
    protected static final Logger LOG = LoggerFactory.getLogger(ContainerProvisionerThreads.class);

    public static void main(String[] argv) throws Exception {
        ContainerProvisionerThreads containerProvisionerThreads = new ContainerProvisionerThreads(argv);
        containerProvisionerThreads.startThreads();
    }

    private ContainerProvisionerThreads(String[] argv) throws IOException {
        super();
        this.testSpec = super.parser.accepts("test", "in test mode, workers are created directly");
        super.parseOptions(argv);
    }

    public void startThreads() throws InterruptedException {
        ExecutorService pool = Executors.newFixedThreadPool(DEFAULT_THREADS);
        ProcessVMOrders processVMOrders = new ProcessVMOrders(this.configFile, this.options.has(this.endlessSpec));
        ProvisionVMs provisionVMs = new ProvisionVMs(this.configFile, this.options.has(this.endlessSpec),
                this.options.has(testSpec));
        CleanupVMs cleanupVMs = new CleanupVMs(this.configFile, this.options.has(this.endlessSpec));
        List<Future<?>> futures = new ArrayList<>();
        futures.add(pool.submit(processVMOrders));
        futures.add(pool.submit(provisionVMs));
        futures.add(pool.submit(cleanupVMs));
        try {
            for (Future<?> future : futures) {
                future.get();
            }
        } catch (InterruptedException | ExecutionException ex) {
            log.error(ex.toString());
            throw new RuntimeException(ex);
        } finally {
            pool.shutdown();
        }
    }

    /**
     * This dequeues the VM requests from the VM queue and stages them in the DB as pending so I can keep a count of what's
     * running/pending/finished.
     */
    private static class ProcessVMOrders implements Callable<Void> {

        protected static final Logger LOG = LoggerFactory.getLogger(ProcessVMOrders.class);
        private final boolean endless;
        private final String config;

        public ProcessVMOrders(String config, boolean endless) {
            this.endless = endless;
            this.config = config;
        }

        @Override
        public Void call() throws IOException, TimeoutException {
            Channel vmChannel = null;
            try {

                HierarchicalINIConfiguration settings = Utilities.parseConfig(config);

                String queueName = settings.getString(Constants.RABBIT_QUEUE_NAME);

                // read from
                vmChannel = Utilities.setupQueue(settings, queueName + "_vms");

                // writes to DB as well
                PostgreSQL db = new PostgreSQL(settings);

                QueueingConsumer consumer = new QueueingConsumer(vmChannel);
                vmChannel.basicConsume(queueName + "_vms", false, consumer);

                // TODO: need threads that each read from orders and another that reads results
                do {
                    LOG.debug("CHECKING FOR NEW VM ORDER!");
                    QueueingConsumer.Delivery delivery = consumer.nextDelivery(FIVE_SECOND_IN_MILLISECONDS);
                    if (delivery == null) {
                        continue;
                    }
                    // jchannel.basicAck(delivery.getEnvelope().getDeliveryTag(), false);
                    String message = new String(delivery.getBody(), StandardCharsets.UTF_8);
                    LOG.info(" [x] Received New VM Request '" + message + "'");

                    // now parse it as a VM order
                    Provision p = new Provision();
                    p.fromJSON(message);
                    p.setState(ProvisionState.PENDING);

                    // puts it into the DB so I can count it in another thread
                    db.createProvision(p);
                    vmChannel.basicAck(delivery.getEnvelope().getDeliveryTag(), false);
                } while (endless);

            } catch (IOException ex) {
                throw new RuntimeException(ex);
            } catch (InterruptedException | ShutdownSignalException | ConsumerCancelledException ex) {
                throw new RuntimeException(ex);
            } finally {
                if (vmChannel != null) {
                    vmChannel.close();
                    vmChannel.getConnection().close();
                }
            }
            return null;
        }
    }

    /**
     * This examines the provision table in the DB to identify the number of running VMs. It then figures out if the number running is less
     * than the max number. If so it picks the oldest pending, switches to running, and launches a worker thread.
     */
    private static class ProvisionVMs implements Callable<Void> {
        protected static final Logger LOG = LoggerFactory.getLogger(ProvisionVMs.class);
        private long maxWorkers = 0;
        private final String configFile;
        private final boolean endless;
        private final boolean testMode;

        public ProvisionVMs(String configFile, boolean endless, boolean testMode) {
            this.configFile = configFile;
            this.endless = endless;
            this.testMode = testMode;
        }

        @Override
        public Void call() throws Exception {
            try {

                HierarchicalINIConfiguration settings = Utilities.parseConfig(configFile);
                if (!settings.containsKey(Constants.PROVISION_MAX_RUNNING_CONTAINERS)) {
                    LOG.info("No max_running_containers specified, skipping provision ");
                    return null;
                }

                // writes to DB as well
                PostgreSQL db = new PostgreSQL(settings);

                // TODO: need threads that each read from orders and another that reads results
                do {

                    // System.out.println("CHECKING RUNNING VMs");

                    // read from DB
                    long numberRunningContainers = db.getJobs(JobState.PENDING).size();
                    long numberPendingContainers = db.getJobs(JobState.RUNNING).size();

                    if (testMode) {
                        LOG.debug("  CHECKING NUMBER OF RUNNING: " + numberRunningContainers);
                        maxWorkers = settings.getLong(Constants.PROVISION_MAX_RUNNING_CONTAINERS);

                        // if this is true need to launch another container
                        if (numberRunningContainers < maxWorkers && numberPendingContainers > 0) {

                            LOG.info("  RUNNING CONTAINERS < " + maxWorkers + " SO WILL LAUNCH VM");

                            // TODO: this will obviously get much more complicated when integrated with Youxia launch VM
                            // fake a uuid
                            String uuid = UUID.randomUUID().toString().toLowerCase();
                            // now launch the VM... doing this after the update above to prevent race condition if the worker signals
                            // finished
                            // before it's marked as pending
                            new WorkerRunnable(configFile, uuid, 1).run();
                            LOG.info("\n\n\nI LAUNCHED A WORKER THREAD FOR VM " + uuid
                                    + " AND IT'S RELEASED!!!\n\n");
                        }
                    } else {
                        long requiredVMs = numberRunningContainers + numberPendingContainers;
                        // cap the number of VMs
                        LOG.info("  Desire for " + requiredVMs + " VMs");
                        requiredVMs = Math.min(requiredVMs,
                                settings.getLong(Constants.PROVISION_MAX_RUNNING_CONTAINERS, Integer.MAX_VALUE));
                        LOG.info("  Capped at " + requiredVMs + " VMs");
                        if (requiredVMs > 0) {
                            String param = settings.getString(Constants.PROVISION_YOUXIA_DEPLOYER);
                            CommandLine parse = CommandLine.parse("dummy " + (param == null ? "" : param));
                            List<String> arguments = new ArrayList<>();
                            arguments.addAll(Arrays.asList(parse.getArguments()));
                            arguments.add("--total-nodes-num");
                            arguments.add(String.valueOf(requiredVMs));
                            String[] toArray = arguments.toArray(new String[arguments.size()]);
                            LOG.info("Running youxia deployer with following parameters:"
                                    + Arrays.toString(toArray));
                            // need to make sure reaper and deployer do not overlap
                            synchronized (ContainerProvisionerThreads.class) {
                                try {
                                    Deployer.main(toArray);
                                } catch (Exception e) {
                                    LOG.error("Youxia deployer threw the following exception", e);
                                    // call the reaper to do cleanup when deployment fails
                                    runReaper(settings, null, null);
                                }
                            }
                        }
                        if (endless) {
                            // lengthen time to allow cleanup queue to purge
                            Thread.sleep(TWO_MINUTE_IN_MILLISECONDS);
                        }
                    }
                } while (endless);

            } catch (ShutdownSignalException | ConsumerCancelledException ex) {
                throw new RuntimeException(ex);
            }
            return null;
        }

    }

    /**
     * This keeps an eye on the results queue. It updates the database with finished jobs. Presumably it should also kill VMs.
     */
    private static class CleanupVMs implements Callable<Void> {
        protected static final Logger LOG = LoggerFactory.getLogger(CleanupVMs.class);
        private final String configFile;
        private final boolean endless;

        public CleanupVMs(String configFile, boolean endless) {
            this.configFile = configFile;
            this.endless = endless;
        }

        @Override
        public Void call() throws Exception {
            Channel resultsChannel = null;
            try {

                HierarchicalINIConfiguration settings = Utilities.parseConfig(configFile);

                String queueName = settings.getString(Constants.RABBIT_QUEUE_NAME);
                final String resultQueueName = queueName + "_results";

                // read from
                resultsChannel = Utilities.setupExchange(settings, resultQueueName);
                // this declares a queue exchange where multiple consumers get the same message:
                // https://www.rabbitmq.com/tutorials/tutorial-three-java.html
                String resultsQueue = Utilities.setupQueueOnExchange(resultsChannel, queueName, "CleanupVMs");
                resultsChannel.queueBind(resultsQueue, resultQueueName, "");
                QueueingConsumer resultsConsumer = new QueueingConsumer(resultsChannel);
                resultsChannel.basicConsume(resultsQueue, false, resultsConsumer);

                // writes to DB as well
                PostgreSQL db = new PostgreSQL(settings);

                boolean reapFailedWorkers = settings.getBoolean(Constants.PROVISION_REAP_FAILED_WORKERS, false);

                // TODO: need threads that each read from orders and another that reads results
                do {

                    LOG.debug("CHECKING FOR VMs TO REAP!");

                    QueueingConsumer.Delivery delivery = resultsConsumer.nextDelivery(FIVE_SECOND_IN_MILLISECONDS);
                    if (delivery == null) {
                        continue;
                    }
                    // jchannel.basicAck(delivery.getEnvelope().getDeliveryTag(), false);
                    String message = new String(delivery.getBody(), StandardCharsets.UTF_8);
                    LOG.debug(" [x] RECEIVED RESULT MESSAGE - ContainerProvisioner: '" + message + "'");

                    // now parse it as JSONObj
                    Status status = new Status().fromJSON(message);

                    // in end states, keep a copy of the results
                    if (status.getState() == StatusState.SUCCESS || status.getState() == StatusState.FAILED) {
                        db.updateJobMessage(status.getJobUuid(), status.getStdout(), status.getStderr());
                    }

                    if (Utilities.JOB_MESSAGE_TYPE.equals(status.getType())) {
                        // now update that DB record to be exited
                        // this is acutally finishing the VM and not the work
                        if (status.getState() == StatusState.SUCCESS) {
                            // finishing the container means a success status
                            // this is where it reaps, the job status message also contains the UUID for the VM
                            db.finishContainer(status.getVmUuid());
                            synchronized (ContainerProvisionerThreads.class) {
                                runReaper(settings, status.getIpAddress(), status.getVmUuid());
                            }
                        } else if (reapFailedWorkers && status.getState() == StatusState.FAILED) {
                            // reaped failed workers need to be set to the failed state
                            ProvisionState provisionState = ProvisionState.FAILED;
                            db.updateProvisionByJobUUID(status.getJobUuid(), status.getVmUuid(), provisionState,
                                    status.getIpAddress());
                            synchronized (ContainerProvisionerThreads.class) {
                                runReaper(settings, status.getIpAddress(), status.getVmUuid());
                            }
                        } else if (status.getState() == StatusState.RUNNING
                                || status.getState() == StatusState.FAILED
                                || status.getState() == StatusState.PENDING
                                || status.getState() == StatusState.PROVISIONING) {
                            // deal with running, failed, pending, provisioning
                            // convert from provision state to statestate
                            ProvisionState provisionState = ProvisionState.valueOf(status.getState().toString());
                            db.updateProvisionByJobUUID(status.getJobUuid(), status.getVmUuid(), provisionState,
                                    status.getIpAddress());
                        }
                    }
                    resultsChannel.basicAck(delivery.getEnvelope().getDeliveryTag(), false);
                } while (endless);

            } catch (IOException ex) {
                LOG.error("CleanupVMs threw the following exception", ex);
                throw new RuntimeException(ex);
            } catch (InterruptedException | ShutdownSignalException | ConsumerCancelledException ex) {
                LOG.error("CleanupVMs threw the following exception", ex);
                throw new RuntimeException(ex);
            } catch (Exception ex) {
                LOG.error("CleanupVMs threw the following exception", ex);
                throw new RuntimeException(ex);
            } finally {
                if (resultsChannel != null) {
                    resultsChannel.close();
                    resultsChannel.getConnection().close();
                }
            }
            return null;
        }
    }

    /**
     * run the reaper
     *
     * @param settings
     * @param ipAddress
     *            specify an ip address (otherwise cleanup only failed deployments)
     * @throws JsonIOException
     * @throws IOException
     */
    private static void runReaper(HierarchicalINIConfiguration settings, String ipAddress, String vmName)
            throws IOException {
        String param = settings.getString(Constants.PROVISION_YOUXIA_REAPER);
        CommandLine parse = CommandLine.parse("dummy " + (param == null ? "" : param));
        List<String> arguments = new ArrayList<>();
        arguments.addAll(Arrays.asList(parse.getArguments()));

        arguments.add("--kill-list");
        // create a json file with the one targetted ip address
        Gson gson = new Gson();
        // we can't use the full set of database records because unlike Amazon, OpenStack reuses private ip addresses (very quickly too)
        // String[] successfulVMAddresses = db.getSuccessfulVMAddresses();
        String[] successfulVMAddresses = new String[] {};
        if (ipAddress != null) {
            successfulVMAddresses = new String[] { ipAddress, vmName };
        }
        LOG.info("Kill list contains: " + Arrays.asList(successfulVMAddresses));
        Path createTempFile = Files.createTempFile("target", "json");
        try (BufferedWriter bw = new BufferedWriter(
                new OutputStreamWriter(new FileOutputStream(createTempFile.toFile()), StandardCharsets.UTF_8))) {
            gson.toJson(successfulVMAddresses, bw);
        }
        arguments.add(createTempFile.toAbsolutePath().toString());

        String[] toArray = arguments.toArray(new String[arguments.size()]);
        LOG.info("Running youxia reaper with following parameters:" + Arrays.toString(toArray));
        // need to make sure reaper and deployer do not overlap

        try {
            Reaper.main(toArray);
        } catch (Exception e) {
            LOG.error("Youxia reaper threw the following exception", e);
        }
    }
}