Example usage for org.apache.hadoop.conf Configuration get

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration get.

Prototype

public String get(String name) 

Source Link

Document

Get the value of the name property, null if no such property exists.

Usage

From source file:com.alibaba.wasp.master.FMasterCommandLine.java

License:Apache License

private int startMaster() {
    Configuration conf = getConf();
    try {//from w  ww. j a v a  2s.c o  m
        // If 'local', defer to LocalWaspCluster instance. Starts master
        // and fserver both in the one JVM.
        if (LocalWaspCluster.isLocal(conf)) {
            final MiniZooKeeperCluster zooKeeperCluster = new MiniZooKeeperCluster();
            File zkDataPath = new File(conf.get(FConstants.ZOOKEEPER_DATA_DIR));
            int zkClientPort = conf.getInt(FConstants.ZOOKEEPER_CLIENT_PORT, 0);
            if (zkClientPort == 0) {
                throw new IOException("No config value for " + FConstants.ZOOKEEPER_CLIENT_PORT);
            }
            zooKeeperCluster.setDefaultClientPort(zkClientPort);
            int clientPort = zooKeeperCluster.startup(zkDataPath);
            if (clientPort != zkClientPort) {
                String errorMsg = "Could not start ZK at requested port of " + zkClientPort
                        + ".  ZK was started at port: " + clientPort
                        + ".  Aborting as clients (e.g. shell) will not be able to find " + "this ZK quorum.";
                System.err.println(errorMsg);
                throw new IOException(errorMsg);
            }
            conf.set(FConstants.ZOOKEEPER_CLIENT_PORT, Integer.toString(clientPort));
            // Need to have the zk cluster shutdown when master is shutdown.
            // Run a subclass that does the zk cluster shutdown on its way out.
            LocalWaspCluster cluster = new LocalWaspCluster(conf, 1, 1, LocalFMaster.class, FServer.class);
            ((LocalFMaster) cluster.getMaster(0)).setZKCluster(zooKeeperCluster);
            cluster.startup();
            waitOnMasterThreads(cluster);
        } else {
            FMaster master = FMaster.constructMaster(masterClass, conf);
            if (master.isStopped()) {
                LOG.info("Won't bring the Master up as a shutdown is requested");
                return -1;
            }
            master.start();
            master.join();
            if (master.isAborted())
                throw new RuntimeException("FMaster Aborted");
        }
    } catch (Throwable t) {
        LOG.error("Failed to start master", t);
        return -1;
    }
    return 0;
}

---

From source file:com.alibaba.wasp.util.WaspConfTool.java

License:Apache License

public static void main(String args[]) {
    if (args.length < 1) {
        System.err.println("Usage: WaspConfTool <CONFIGURATION_KEY>");
        System.exit(1);/*from   w ww.ja va  2 s.c om*/
        return;
    }

    Configuration conf = WaspConfiguration.create();
    System.out.println(conf.get(args[0]));
}

---

From source file:com.alibaba.wasp.zookeeper.ZKConfig.java

License:Apache License

/**
 * Make a Properties object holding ZooKeeper config equivalent to zoo.cfg.
 * If there is a zoo.cfg in the classpath, simply read it in. Otherwise parse
 * the corresponding config options from the Wasp XML configs and generate
 * the appropriate ZooKeeper properties.
 * @param conf Configuration to read from.
 * @return Properties holding mappings representing ZooKeeper zoo.cfg file.
 *//*from  w  ww  .  jav a 2s .c o m*/
public static Properties makeZKProps(Configuration conf) {
    // First check if there is a zoo.cfg in the CLASSPATH. If so, simply read
    // it and grab its configuration properties.
    ClassLoader cl = FQuorumPeer.class.getClassLoader();
    final InputStream inputStream = cl.getResourceAsStream(FConstants.ZOOKEEPER_CONFIG_NAME);
    if (inputStream != null) {
        try {
            return parseZooCfg(conf, inputStream);
        } catch (IOException e) {
            LOG.warn("Cannot read " + FConstants.ZOOKEEPER_CONFIG_NAME + ", loading from XML files", e);
        }
    }

    // Otherwise, use the configuration options from Wasp's XML files.
    Properties zkProperties = new Properties();

    // Directly map all of the wasp.zookeeper.property.KEY properties.
    for (Entry<String, String> entry : conf) {
        String key = entry.getKey();
        if (key.startsWith(FConstants.ZK_CFG_PROPERTY_PREFIX)) {
            String zkKey = key.substring(FConstants.ZK_CFG_PROPERTY_PREFIX_LEN);
            String value = entry.getValue();
            // If the value has variables substitutions, need to do a get.
            if (value.contains(VARIABLE_START)) {
                value = conf.get(key);
            }
            zkProperties.put(zkKey, value);
        }
    }

    // If clientPort is not set, assign the default.
    if (zkProperties.getProperty(FConstants.CLIENT_PORT_STR) == null) {
        zkProperties.put(FConstants.CLIENT_PORT_STR, FConstants.DEFAULT_ZOOKEPER_CLIENT_PORT);
    }

    // Create the server.X properties.
    int peerPort = conf.getInt("wasp.zookeeper.peerport", 2888);
    int leaderPort = conf.getInt("wasp.zookeeper.leaderport", 3888);

    final String[] serverHosts = conf.getStrings(FConstants.ZOOKEEPER_QUORUM, FConstants.LOCALHOST);
    for (int i = 0; i < serverHosts.length; ++i) {
        String serverHost = serverHosts[i];
        String address = serverHost + ":" + peerPort + ":" + leaderPort;
        String key = "server." + i;
        zkProperties.put(key, address);
    }

    return zkProperties;
}

---

From source file:com.alibaba.wasp.zookeeper.ZKConfig.java

License:Apache License

/**
 * Parse ZooKeeper's zoo.cfg, injecting Wasp Configuration variables in.
 * This method is used for testing so we can pass our own InputStream.
 * @param conf WaspConfiguration to use for injecting variables.
 * @param inputStream InputStream to read from.
 * @return Properties parsed from config stream with variables substituted.
 * @throws java.io.IOException if anything goes wrong parsing config
 *//*from   www .  ja  va2s. c  om*/
public static Properties parseZooCfg(Configuration conf, InputStream inputStream) throws IOException {
    Properties properties = new Properties();
    try {
        properties.load(inputStream);
    } catch (IOException e) {
        final String msg = "fail to read properties from " + FConstants.ZOOKEEPER_CONFIG_NAME;
        LOG.fatal(msg);
        throw new IOException(msg, e);
    }
    for (Entry<Object, Object> entry : properties.entrySet()) {
        String value = entry.getValue().toString().trim();
        String key = entry.getKey().toString().trim();
        StringBuilder newValue = new StringBuilder();
        int varStart = value.indexOf(VARIABLE_START);
        int varEnd = 0;
        while (varStart != -1) {
            varEnd = value.indexOf(VARIABLE_END, varStart);
            if (varEnd == -1) {
                String msg = "variable at " + varStart + " has no end marker";
                LOG.fatal(msg);
                throw new IOException(msg);
            }
            String variable = value.substring(varStart + VARIABLE_START_LENGTH, varEnd);

            String substituteValue = System.getProperty(variable);
            if (substituteValue == null) {
                substituteValue = conf.get(variable);
            }
            if (substituteValue == null) {
                String msg = "variable " + variable + " not set in system property " + "or wasp configs";
                LOG.fatal(msg);
                throw new IOException(msg);
            }

            newValue.append(substituteValue);

            varEnd += VARIABLE_END_LENGTH;
            varStart = value.indexOf(VARIABLE_START, varEnd);
        }
        // Special case for 'wasp.cluster.distributed' property being 'true'
        if (key.startsWith("server.")) {
            boolean mode = conf.getBoolean(FConstants.CLUSTER_DISTRIBUTED,
                    FConstants.DEFAULT_CLUSTER_DISTRIBUTED);
            if (mode == FConstants.CLUSTER_IS_DISTRIBUTED && value.startsWith(FConstants.LOCALHOST)) {
                String msg = "The server in zoo.cfg cannot be set to localhost "
                        + "in a fully-distributed setup because it won't be reachable. "
                        + "See \"Getting Started\" for more information.";
                LOG.fatal(msg);
                throw new IOException(msg);
            }
        }
        newValue.append(value.substring(varEnd));
        properties.setProperty(key, newValue.toString());
    }
    return properties;
}

---

From source file:com.alibaba.wasp.zookeeper.ZKUtil.java

License:Apache License

/**
 * Get the key to the ZK ensemble for this configuration and append a name at
 * the end/*from w  ww.  ja  v a2  s  . c o  m*/
 *
 * @param conf
 *          Configuration to use to build the key
 * @param name
 *          Name that should be appended at the end if not empty or null
 * @return ensemble key with a name (if any)
 */
public static String getZooKeeperClusterKey(Configuration conf, String name) {
    String ensemble = conf.get(HConstants.ZOOKEEPER_QUORUM.replaceAll("[\\t\\n\\x0B\\f\\r]", ""));
    StringBuilder builder = new StringBuilder(ensemble);
    builder.append(":");
    builder.append(conf.get(HConstants.ZOOKEEPER_CLIENT_PORT));
    builder.append(":");
    builder.append(conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT));
    if (name != null && !name.isEmpty()) {
        builder.append(",");
        builder.append(name);
    }
    return builder.toString();
}

---

From source file:com.alimama.quanjingmonitor.kmeans.KMeansClusterCombiner.java

License:Apache License

@Override
protected void setup(Context context) throws IOException, InterruptedException {
    super.setup(context);
    this.clusters.clear();

    Configuration conf = context.getConfiguration();
    this.rep = conf.getInt(KMeansDriver.CLUSTER_CONVERGENCE_ABTEST_REP, 2);

    try {//  www.  j  a  va  2s.c om

        String clusterPath = conf.get(KMeansDriver.CLUSTER_PATH_KEY);
        if (clusterPath != null && clusterPath.length() > 0) {
            KmeansPublic.configureWithClusterInfo(conf, new Path(clusterPath), clusters);
            if (clusters.isEmpty()) {
                throw new IllegalStateException("No clusters found. Check your -c path.");
            }
            this.setClusterMap(clusters);
        }
    } catch (Throwable e) {
        throw new IllegalStateException(e);
    }
}

---

From source file:com.alimama.quanjingmonitor.kmeans.KMeansClusterMapper.java

License:Apache License

@Override
protected void setup(Context context) throws IOException, InterruptedException {
    super.setup(context);
    this.clusters.clear();

    Configuration conf = context.getConfiguration();
    parse.setup(conf);/*from w w  w  . j a  v a 2s.c  om*/

    try {

        String clusterPath = conf.get(KMeansDriver.CLUSTER_PATH_KEY);
        if (clusterPath != null && clusterPath.length() > 0) {
            KmeansPublic.configureWithClusterInfo(conf, new Path(clusterPath), clusters);
            if (clusters.isEmpty()) {
                throw new IllegalStateException("No clusters found. Check your -c path.");
            }
        }
    } catch (Throwable e) {
        throw new IllegalStateException(e);
    }
}

---

From source file:com.alimama.quanjingmonitor.kmeans.KMeansClusterReduce.java

License:Apache License

@Override
protected void setup(Context context) throws IOException, InterruptedException {
    super.setup(context);
    this.clusters.clear();

    Configuration conf = context.getConfiguration();
    parse.setup(conf);//from w w w  . j  av a  2  s.  co  m
    this.rep = conf.getInt(KMeansDriver.CLUSTER_CONVERGENCE_ABTEST_REP, 2);
    try {

        String clusterPath = conf.get(KMeansDriver.CLUSTER_PATH_KEY);
        if (clusterPath != null && clusterPath.length() > 0) {
            KmeansPublic.configureWithClusterInfo(conf, new Path(clusterPath), clusters);
            if (clusters.isEmpty()) {
                throw new IllegalStateException("No clusters found. Check your -c path.");
            }
            this.setClusterMap(clusters);
        }
    } catch (Throwable e) {
        throw new IllegalStateException(e);
    }
}

---

From source file:com.aliyun.emr.example.EMapReduceOSSUtil.java

License:Apache License

public static String buildOSSCompleteUri(String oriUri, Configuration conf) {
    return buildOSSCompleteUri(oriUri, conf.get("fs.oss.accessKeyId"), conf.get("fs.oss.accessKeySecret"),
            conf.get("fs.oss.endpoint"));
}

---

From source file:com.aliyun.fs.oss.TestAliyunOSSFileSystemStore.java

License:Apache License

@BeforeClass
public static void checkSettings() throws Exception {
    Configuration conf = new Configuration();
    assumeNotNull(conf.get("fs.oss.accessKeyId"));
    assumeNotNull(conf.get("fs.oss.accessKeySecret"));
    assumeNotNull(conf.get("test.fs.oss.name"));
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• »