Example usage for org.apache.hadoop.conf Configuration set

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration set.

Prototype

public void set(String name, String value)

Source Link

Document

Set the value of the name property.

Usage

From source file:com.alibaba.jstorm.hbase.AbstractHBaseClient.java

License:Apache License

public Configuration makeConf(Map stormConf) {
    Configuration hbaseConf = HBaseConfiguration.create();
    String hbaseQuorum = (String) stormConf.get(HBASE_QUORUM_CONF_KEY);
    hbaseConf.set(HBASE_QUORUM_KEY, hbaseQuorum);

    String hbasePort = stormConf.get(HBASE_PORT_CONF_KEY) + "";
    hbaseConf.set(HBASE_PORT_KEY, hbasePort);

    String hbaseParent = (String) stormConf.get(HBASE_ZK_PARENT_CONF_KEY);
    hbaseConf.set(HBASE_ZK_PARENT_KEY, hbaseParent);

    return hbaseConf;
}

From source file:com.alibaba.jstorm.hdfs.common.security.AutoHDFS.java

License:Apache License

private void login(Configuration configuration) throws IOException {
    configuration.set(STORM_KEYTAB_FILE_KEY, this.hdfsKeyTab);
    configuration.set(STORM_USER_NAME_KEY, this.hdfsPrincipal);
    SecurityUtil.login(configuration, STORM_KEYTAB_FILE_KEY, STORM_USER_NAME_KEY);

    LOG.info("Logged into hdfs with principal {}", this.hdfsPrincipal);
}

From source file:com.alibaba.jstorm.hdfs.common.security.HdfsSecurityUtil.java

License:Apache License

public static void login(Map conf, Configuration hdfsConfig) throws IOException {
    //If AutoHDFS is specified, do not attempt to login using keytabs, only kept for backward compatibility.
    if (conf.get(TOPOLOGY_AUTO_CREDENTIALS) == null
            || (!(((List) conf.get(TOPOLOGY_AUTO_CREDENTIALS)).contains(AutoHDFS.class.getName()))
                    && !(((List) conf.get(TOPOLOGY_AUTO_CREDENTIALS)).contains(AutoTGT.class.getName())))) {
        if (UserGroupInformation.isSecurityEnabled()) {
            // compareAndSet added because of https://issues.apache.org/jira/browse/STORM-1535
            if (isLoggedIn.compareAndSet(false, true)) {
                LOG.info("Logging in using keytab as AutoHDFS is not specified for "
                        + TOPOLOGY_AUTO_CREDENTIALS);
                String keytab = (String) conf.get(STORM_KEYTAB_FILE_KEY);
                if (keytab != null) {
                    hdfsConfig.set(STORM_KEYTAB_FILE_KEY, keytab);
                }/*from  w w w .j av  a 2s.  c  o m*/
                String userName = (String) conf.get(STORM_USER_NAME_KEY);
                if (userName != null) {
                    hdfsConfig.set(STORM_USER_NAME_KEY, userName);
                }
                SecurityUtil.login(hdfsConfig, STORM_KEYTAB_FILE_KEY, STORM_USER_NAME_KEY);
            }
        }
    }
}

From source file:com.alibaba.wasp.conf.WaspConfiguration.java

License:Apache License

/**
 * Merge two configurations./*from   w  ww .j  a v a2s. co m*/
 * @param destConf the configuration that will be overwritten with items
 *                 from the srcConf
 * @param srcConf the source configuration
 **/
public static void merge(Configuration destConf, Configuration srcConf) {
    for (Entry<String, String> e : srcConf) {
        destConf.set(e.getKey(), e.getValue());
    }
}

From source file:com.alibaba.wasp.LocalWaspCluster.java

License:Apache License

/**
 * Constructor./*from  ww w .jav  a2s .c om*/
 *
 * @param conf
 *          Configuration to use. Post construction has the master's address.
 * @param noMasters
 *          Count of masters to start.
 * @param noFServers
 *          Count of fservers to start.
 * @param masterClass
 * @param fserverClass
 * @throws java.io.IOException
 */
@SuppressWarnings("unchecked")
public LocalWaspCluster(final Configuration conf, final int noMasters, final int noFServers,
        final Class<? extends FMaster> masterClass, final Class<? extends FServer> fserverClass)
        throws IOException {
    this.conf = conf;
    // Always have masters and fservers come up on port '0' so we don't
    // clash over default ports.
    conf.set(FConstants.MASTER_PORT, "0");
    conf.set(FConstants.FSERVER_PORT, "0");
    this.masterClass = (Class<? extends FMaster>) conf.getClass(FConstants.MASTER_IMPL, masterClass);
    // Start the FMasters.
    for (int i = 0; i < noMasters; i++) {
        addMaster(new Configuration(conf), i);
    }

    // Wait for master active.
    try {
        Thread.sleep(3000);
    } catch (InterruptedException e) {
        throw new IOException(e);
    }

    // Start the FServers.
    this.fserverClass = (Class<? extends FServer>) conf.getClass(FConstants.FSERVER_IMPL, fserverClass);

    for (int i = 0; i < noFServers; i++) {
        addFServer(new Configuration(conf), i);
    }
}

From source file:com.alibaba.wasp.master.balancer.TestDefaultLoadBalancer.java

License:Apache License

@BeforeClass
public static void beforeAllTests() throws Exception {
    Configuration conf = HBaseConfiguration.create();
    conf.set("hbase.entityGroups.slop", "0");
    loadBalancer = new DefaultLoadBalancer();
    loadBalancer.setConf(conf);//  www.  jav a2  s. c  o m
}

From source file:com.alibaba.wasp.master.FMasterCommandLine.java

License:Apache License

private int startMaster() {
    Configuration conf = getConf();
    try {//from w  w  w . j a v a2s . c  om
        // If 'local', defer to LocalWaspCluster instance. Starts master
        // and fserver both in the one JVM.
        if (LocalWaspCluster.isLocal(conf)) {
            final MiniZooKeeperCluster zooKeeperCluster = new MiniZooKeeperCluster();
            File zkDataPath = new File(conf.get(FConstants.ZOOKEEPER_DATA_DIR));
            int zkClientPort = conf.getInt(FConstants.ZOOKEEPER_CLIENT_PORT, 0);
            if (zkClientPort == 0) {
                throw new IOException("No config value for " + FConstants.ZOOKEEPER_CLIENT_PORT);
            }
            zooKeeperCluster.setDefaultClientPort(zkClientPort);
            int clientPort = zooKeeperCluster.startup(zkDataPath);
            if (clientPort != zkClientPort) {
                String errorMsg = "Could not start ZK at requested port of " + zkClientPort
                        + ".  ZK was started at port: " + clientPort
                        + ".  Aborting as clients (e.g. shell) will not be able to find " + "this ZK quorum.";
                System.err.println(errorMsg);
                throw new IOException(errorMsg);
            }
            conf.set(FConstants.ZOOKEEPER_CLIENT_PORT, Integer.toString(clientPort));
            // Need to have the zk cluster shutdown when master is shutdown.
            // Run a subclass that does the zk cluster shutdown on its way out.
            LocalWaspCluster cluster = new LocalWaspCluster(conf, 1, 1, LocalFMaster.class, FServer.class);
            ((LocalFMaster) cluster.getMaster(0)).setZKCluster(zooKeeperCluster);
            cluster.startup();
            waitOnMasterThreads(cluster);
        } else {
            FMaster master = FMaster.constructMaster(masterClass, conf);
            if (master.isStopped()) {
                LOG.info("Won't bring the Master up as a shutdown is requested");
                return -1;
            }
            master.start();
            master.join();
            if (master.isAborted())
                throw new RuntimeException("FMaster Aborted");
        }
    } catch (Throwable t) {
        LOG.error("Failed to start master", t);
        return -1;
    }
    return 0;
}

From source file:com.alibaba.wasp.master.TestWaspRPCException.java

License:Apache License

@Test
public void testRPCException() throws Exception {
    WaspTestingUtility TEST_UTIL = new WaspTestingUtility();
    TEST_UTIL.getHBaseTestingUtility().startMiniZKCluster();
    Configuration conf = TEST_UTIL.getConfiguration();
    conf.set(FConstants.MASTER_PORT, "0");
    TEST_UTIL.getConfiguration().set(FConstants.ZOOKEEPER_QUORUM,
            TEST_UTIL.getConfiguration().get(HConstants.ZOOKEEPER_QUORUM));
    TEST_UTIL.getConfiguration().set(FConstants.ZOOKEEPER_CLIENT_PORT,
            TEST_UTIL.getConfiguration().get(HConstants.ZOOKEEPER_CLIENT_PORT));
    FMaster hm = new FMaster(conf);

    ServerName sm = hm.getServerName();/* w  ww  .  j  a  v a 2 s  .com*/
    InetSocketAddress isa = new InetSocketAddress(sm.getHostname(), sm.getPort());
    int i = 0;
    // retry the RPC a few times; we have seen SocketTimeoutExceptions if we
    // try to connect too soon. Retry on SocketTimeoutException.
    while (i < 20) {
        try {
            FMasterMonitorProtocol inf = (FMasterMonitorProtocol) WaspRPC.getProxy(FMasterMonitorProtocol.class,
                    FMasterMonitorProtocol.VERSION, isa, conf, 100 * 10);
            inf.isMasterRunning(null, MasterProtos.IsMasterRunningRequest.getDefaultInstance());
            fail();
        } catch (ServiceException ex) {
            IOException ie = ProtobufUtil.getRemoteException(ex);
            if (ie instanceof RemoteException) {
                ie = ((RemoteException) ie).unwrapRemoteException();
            }
            if (!(ie instanceof SocketTimeoutException)) {
                if (ie.getMessage().indexOf("Server is not running yet") != -1) {
                    return;
                }
            } else {
                System.err.println("Got SocketTimeoutException. Will retry. ");
            }
        } catch (Throwable t) {
            fail("Unexpected throwable: " + t);
        }
        Thread.sleep(100);
        i++;
    }
    fail();
}

From source file:com.alibaba.wasp.MiniWaspCluster.java

License:Apache License

public MiniWaspCluster(Configuration conf, int numMasters, int numFServers,
        Class<? extends FMaster> masterClass, Class<? extends MiniWaspClusterFServer> fserverClass)
        throws IOException, InterruptedException {
    super(conf);// w w  w  .ja v a2 s .c  om
    conf.set(FConstants.MASTER_PORT, "0");
    init(numMasters, numFServers, masterClass, fserverClass);
    this.initialClusterStatus = getClusterStatus();
}

From source file:com.alibaba.wasp.util.Utils.java

License:Apache License

public static Configuration convertPropertiesToConfiguration(Properties properties) {
    Configuration conf = WaspConfiguration.create();
    for (Map.Entry entry : properties.entrySet()) {
        if (entry.getKey() instanceof String) {
            Object value = properties.get(entry.getKey());
            if (value instanceof String) {
                conf.set((String) entry.getKey(), (String) value);
            }/*from  w w w.j  a  va 2 s . com*/
        }
    }
    return conf;
}