com.cloudera.beeswax.Server.java Source code

Java tutorial

Introduction

Here is the source code for com.cloudera.beeswax.Server.java

Source

// Licensed to Cloudera, Inc. under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  Cloudera, Inc. licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.cloudera.beeswax;

import java.io.IOException;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore;
import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.log4j.Logger;
import org.apache.thrift.TProcessor;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.server.TServer;
import org.apache.thrift.server.TThreadPoolServer;
import org.apache.thrift.transport.TServerSocket;
import org.apache.thrift.transport.TServerTransport;
import org.apache.thrift.transport.TTransportException;
import org.apache.thrift.transport.TTransportFactory;

import com.cloudera.beeswax.api.BeeswaxService;
import com.cloudera.beeswax.api.BeeswaxService.Processor;
import com.facebook.fb303.FacebookService;

public class Server {

    private static final Logger LOG = Logger.getLogger(Server.class.getName());
    private static final int USHRT_MAX = 65535;

    private static int mport = -1;
    private static int bport = -1;
    /** Superuser on the HDFS cluster.  We impersonate to create Hive's directories. */
    private static String superUser = "hadoop";
    /** Host and port that desktop runs on */
    private static String dtHost = "";
    private static int dtPort = -1;
    private static boolean dtHttps = false;

    /**
     * Parse command line options.
     *
     * -b <port> specifies the port for beeswax to use.
     * -m <port>, if given, starts the metastore at this port.
     */
    private static void parseArgs(String[] args) throws ParseException {
        Options options = new Options();

        Option metastoreOpt = new Option("m", "metastore", true, "port to use for metastore");
        metastoreOpt.setRequired(false);
        options.addOption(metastoreOpt);

        Option beeswaxOpt = new Option("b", "beeswax", true, "port to use for beeswax");
        beeswaxOpt.setRequired(true);
        options.addOption(beeswaxOpt);

        Option dtHostOpt = new Option("h", "desktop-host", true, "host running desktop");
        dtHostOpt.setRequired(true);
        options.addOption(dtHostOpt);

        Option dtHttpsOpt = new Option("s", "desktop-https", true, "desktop is running https");
        options.addOption(dtHttpsOpt);

        Option dtPortOpt = new Option("p", "desktop-port", true, "port used by desktop");
        dtPortOpt.setRequired(true);
        options.addOption(dtPortOpt);

        Option superUserOpt = new Option("u", "superuser", true, "Username of Hadoop superuser (default: hadoop)");
        superUserOpt.setRequired(false);
        options.addOption(superUserOpt);

        PosixParser parser = new PosixParser();
        CommandLine cmd = parser.parse(options, args);
        if (!cmd.getArgList().isEmpty()) {
            throw new ParseException("Unexpected extra arguments: " + cmd.getArgList());
        }

        for (Option opt : cmd.getOptions()) {
            if (opt.getOpt() == "m") {
                mport = parsePort(opt);
            } else if (opt.getOpt() == "b") {
                bport = parsePort(opt);
            } else if (opt.getOpt() == "u") {
                superUser = opt.getValue();
            } else if (opt.getOpt() == "h") {
                dtHost = opt.getValue();
            } else if (opt.getOpt() == "p") {
                dtPort = parsePort(opt);
            } else if (opt.getOpt() == "s") {
                dtHttps = true;
            }
        }
    }

    private static int parsePort(Option opt) throws ParseException {
        int port = Integer.valueOf(opt.getValue());
        if (port < 0 || port > USHRT_MAX)
            throw new ParseException("Port number must be a number in [0, " + USHRT_MAX + "]");
        return port;
    }

    public static void main(String[] args) throws TTransportException, MetaException, ParseException {
        parseArgs(args);
        createDirectoriesAsNecessary();

        // Start metastore if specified
        if (mport != -1) {
            LOG.info("Starting metastore at port " + mport);
            Thread t = new Thread(new Runnable() {
                @Override
                public void run() {
                    try {
                        serveMeta(mport);
                    } catch (TTransportException e) {
                        e.printStackTrace();
                    } catch (MetaException e) {
                        e.printStackTrace();
                    }
                }
            }, "MetaServerThread");
            t.setDaemon(true);
            t.start();
        }

        // Serve beeswax out of the main thread.
        LOG.info("Starting beeswaxd at port " + bport);
        serveBeeswax(bport);
    }

    /**
     * Hive won't work unless /tmp and /user/hive/warehouse are usable,
     * so we create them for the user.
     */
    private static void createDirectoriesAsNecessary() {
        try {
            HiveConf conf = new HiveConf(Driver.class);
            conf.set("hadoop.ugi", superUser + ",supergroup");
            FileSystem fs = FileSystem.get(conf);
            Path tmpDir = new Path("/tmp");
            Path metaDir = new Path(conf.get(HiveConf.ConfVars.METASTOREWAREHOUSE.varname));
            for (Path dir : new Path[] { tmpDir, metaDir }) {
                if (!fs.exists(dir)) {
                    if (fs.mkdirs(dir)) {
                        fs.setPermission(dir, new FsPermission((short) 0777));
                        LOG.info("Created " + dir + " with world-writable permissions.");
                    } else {
                        LOG.error("Could not create " + dir);
                    }
                }
            }
        } catch (IOException e) {
            LOG.error("Error while trying to check/create /tmp and warehouse directory.", e);
        }
    }

    /**
     * Start the Beeswax server.
     */
    private static void serveBeeswax(int port) throws TTransportException {
        TServerTransport serverTransport = new TServerSocket(port);
        BeeswaxService.Iface impl = new BeeswaxServiceImpl(dtHost, dtPort, dtHttps);
        Processor processor = new BeeswaxService.Processor(impl);
        TThreadPoolServer.Options options = new TThreadPoolServer.Options();
        TServer server = new TThreadPoolServer(processor, serverTransport, new TTransportFactory(),
                new TTransportFactory(), new TBinaryProtocol.Factory(), new TBinaryProtocol.Factory(), options);
        LOG.info("Starting beeswax server on port " + port + ", talking back to Desktop at " + dtHost + ":"
                + dtPort);
        server.serve();
    }

    /**
     * Start the thrift metastore server.
     *
     * Mostly borrowed from org.apache.hadoop.hive.metastore.HiveMetaStore.
     */
    public static void serveMeta(int port) throws MetaException, TTransportException {
        // Verify that we're supposed to run an internal metastore.
        HiveConf conf = new HiveConf(Driver.class);
        if (!conf.getBoolean("hive.metastore.local", true)) {
            String msg = "hive.metastore.local is set to false. The Beeswax internal metastore "
                    + "is not supposed to run.";
            LOG.fatal(msg);
            System.exit(1);
        }

        TServerTransport serverTransport = new TServerSocket(port);
        Iface handler = new HMSHandler("new db based metaserver");
        FacebookService.Processor processor = new ThriftHiveMetastore.Processor(handler);

        TThreadPoolServer.Options options = new TThreadPoolServer.Options();
        options.minWorkerThreads = 10;
        TServer server = new TThreadPoolServer((TProcessor) processor, serverTransport, new TTransportFactory(),
                new TTransportFactory(), new TBinaryProtocol.Factory(), new TBinaryProtocol.Factory(), options);
        HMSHandler.LOG.info("Started the new metaserver on port [" + port + "]...");
        HMSHandler.LOG.info("Options.minWorkerThreads = " + options.minWorkerThreads);
        HMSHandler.LOG.info("Options.maxWorkerThreads = " + options.maxWorkerThreads);
        server.serve();
    }
}