Example usage for org.apache.hadoop.conf Configuration get

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration get.

Prototype

public String get(String name) 

Source Link

Document

Get the value of the name property, null if no such property exists.

Usage

From source file:com.cloudera.lib.util.XConfiguration.java

License:Open Source License

/**
 * Injects configuration key/value pairs from one configuration to another if the key does not exist in the target
 * configuration.//  w w  w .j  av a 2  s  .  c om
 *
 * @param source source configuration.
 * @param target target configuration.
 */
public static void injectDefaults(Configuration source, Configuration target) {
    Check.notNull(source, "source");
    Check.notNull(target, "target");
    for (Map.Entry<String, String> entry : source) {
        if (target.get(entry.getKey()) == null) {
            target.set(entry.getKey(), entry.getValue());
        }
    }
}

---

From source file:com.cloudera.llama.am.MiniClusterNodeMapper.java

License:Apache License

@Override
@SuppressWarnings("unchecked")
public void setConf(Configuration conf) {
    this.conf = conf;
    try {/*from  www .j  a  v  a 2  s. co  m*/
        String str = conf.get(MAPPING_KEY);
        if (str != null) {
            StringReader reader = new StringReader(str);
            Properties props = new Properties();
            props.load(reader);
            dn2nm = new HashMap<String, String>((Map) props);
            nm2dn = new HashMap<String, String>();
            for (Map.Entry<String, String> entry : dn2nm.entrySet()) {
                nm2dn.put(entry.getValue(), entry.getKey());
            }
        } else {
            throw new RuntimeException(
                    FastFormat.format("Mapping property '{}' not set in the configuration", MAPPING_KEY));
        }
    } catch (Throwable ex) {
        throw new RuntimeException(ex);
    }
}

---

From source file:com.cloudera.llama.am.MiniLlama.java

License:Apache License

private Map<String, String> getDataNodeNodeManagerMapping(Configuration conf) throws Exception {
    Map<String, String> map = new HashMap<String, String>();
    DFSClient dfsClient = new DFSClient(new URI(conf.get("fs.defaultFS")), conf);
    DatanodeInfo[] DNs = dfsClient.datanodeReport(HdfsConstants.DatanodeReportType.ALL);
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);/*  w w w.j a  v  a 2 s.  c  o  m*/
    yarnClient.start();
    List<NodeId> nodeIds = getYarnNodeIds(conf);
    if (nodeIds.size() != DNs.length) {
        throw new RuntimeException("Number of DNs and NMs differ, MiniLlama "
                + "node mapping requires them to be equal at startup");
    }
    LOG.info("HDFS/YARN mapping:");
    for (int i = 0; i < DNs.length; i++) {
        String key = DNs[i].getXferAddr();
        NodeId nodeId = nodeIds.get(i);
        String value = nodeId.getHost() + ":" + nodeId.getPort();
        map.put(key, value);
        LOG.info("  DN/NM: " + key + "/" + value);
    }
    yarnClient.stop();
    nodes = map.size();
    verifySingleHost(map.keySet(), "DataNode");
    verifySingleHost(map.values(), "NodeManager");
    return map;
}

---

From source file:com.cloudera.llama.am.MiniLlama.java

License:Apache License

private Configuration startMiniHadoop() throws Exception {
    int clusterNodes = getConf().getInt(MINI_CLUSTER_NODES_KEY, 1);
    if (System.getProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA) == null) {
        String testBuildData = new File("target").getAbsolutePath();
        System.setProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA, testBuildData);
    }/*w  w w .  j a v  a  2s .com*/
    //to trigger hdfs-site.xml registration as default resource
    new HdfsConfiguration();
    Configuration conf = new YarnConfiguration();
    String llamaProxyUser = System.getProperty("user.name");
    conf.set("hadoop.security.authentication", "simple");
    conf.set("hadoop.proxyuser." + llamaProxyUser + ".hosts", "*");
    conf.set("hadoop.proxyuser." + llamaProxyUser + ".groups", "*");
    String[] userGroups = new String[] { "g" };
    UserGroupInformation.createUserForTesting(llamaProxyUser, userGroups);

    int hdfsPort = 0;
    String fsUri = conf.get("fs.defaultFS");
    if (fsUri != null && !fsUri.equals("file:///")) {
        int i = fsUri.lastIndexOf(":");
        if (i > -1) {
            try {
                hdfsPort = Integer.parseInt(fsUri.substring(i + 1));
            } catch (Exception ex) {
                throw new RuntimeException(
                        "Could not parse port from Hadoop's " + "'fs.defaultFS property: " + fsUri);
            }
        }
    }
    miniHdfs = new MiniDFSCluster(hdfsPort, conf, clusterNodes, !skipDfsFormat, true, null, null);
    miniHdfs.waitActive();
    conf = miniHdfs.getConfiguration(0);
    miniYarn = new MiniYARNCluster("minillama", clusterNodes, 1, 1);
    conf.setBoolean(YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, true);
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 0);
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0);

    miniYarn.init(conf);
    miniYarn.start();
    conf = miniYarn.getConfig();

    ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
    return conf;
}

---

From source file:com.cloudera.llama.am.TestSecureLlamaAMThriftServer.java

License:Apache License

@Override
protected Configuration createCallbackConfiguration() throws Exception {
    ServerConfiguration cConf = new NotificationServerConfiguration();
    Configuration conf = super.createCallbackConfiguration();
    String confDir = conf.get(ServerConfiguration.CONFIG_DIR_KEY);
    ParamChecker.notEmpty(confDir, "missing confDir");

    conf.setBoolean(cConf.getPropertyName(ServerConfiguration.SECURITY_ENABLED_KEY), true);
    File keytab = new File(confDir, "notification.keytab");
    miniKdc.createPrincipal(keytab, "notification/localhost");
    conf.set(cConf.getPropertyName(ServerConfiguration.KEYTAB_FILE_KEY), keytab.getAbsolutePath());
    conf.set(cConf.getPropertyName(ServerConfiguration.SERVER_PRINCIPAL_NAME_KEY), "notification/localhost");
    conf.set(cConf.getPropertyName(ServerConfiguration.SERVER_ADDRESS_KEY), "localhost:0");
    return conf;/*from   w ww  .ja v  a  2  s.  c  o  m*/
}

---

From source file:com.cloudera.llama.am.TestSecureLlamaAMThriftServer.java

License:Apache License

@Override
protected Configuration createLlamaConfiguration() throws Exception {
    Configuration conf = super.createLlamaConfiguration();
    String confDir = conf.get(ServerConfiguration.CONFIG_DIR_KEY);
    ParamChecker.notEmpty(confDir, "missing confDir");
    conf.setBoolean(amConf.getPropertyName(ServerConfiguration.SECURITY_ENABLED_KEY), true);
    File keytab = new File(confDir, "llama.keytab");
    miniKdc.createPrincipal(keytab, "llama/localhost");
    conf.set(amConf.getPropertyName(ServerConfiguration.KEYTAB_FILE_KEY), keytab.getAbsolutePath());
    conf.set(amConf.getPropertyName(ServerConfiguration.SERVER_PRINCIPAL_NAME_KEY), "llama/localhost");
    conf.set(amConf.getPropertyName(ServerConfiguration.SERVER_ADDRESS_KEY), "localhost:0");
    conf.set(amConf.getPropertyName(ServerConfiguration.NOTIFICATION_PRINCIPAL_NAME_KEY), "notification");
    return conf;/*from www.  j a  va2s .c o  m*/
}

---

From source file:com.cloudera.llama.server.ThriftServer.java

License:Apache License

@Override
public void setConf(Configuration conf) {
    if (conf.get(ServerConfiguration.CONFIG_DIR_KEY) == null) {
        throw new RuntimeException(FastFormat.format("Required configuration property '{}' missing",
                ServerConfiguration.CONFIG_DIR_KEY));
    }/*from  w w w.  j  a v a2s .c om*/
    super.setConf(conf);
    sConf = ReflectionUtils.newInstance(serverConfClass, conf);
}

---

From source file:com.cloudera.oryx.als.computation.IDMappingState.java

License:Open Source License

public IDMappingState(Configuration conf) throws IOException {
    this.idMapping = readIDMapping(conf.get(ID_MAPPING_KEY));
}

---

From source file:com.cloudera.oryx.als.computation.iterate.row.YState.java

License:Open Source License

synchronized void initialize(TaskInputOutputContext<?, ?, ?, ?> context, int currentPartition,
        int numPartitions) {

    Configuration conf = context.getConfiguration();

    LongSet expectedIDs;//  w  w w.  j  a  v  a 2  s .c  o  m
    try {
        expectedIDs = ComputationDataUtils.readExpectedIDsFromPartition(currentPartition, numPartitions,
                conf.get(RowStep.POPULAR_KEY), context, conf);
    } catch (IOException e) {
        throw new IllegalStateException(e);
    }

    String yKey = conf.get(RowStep.Y_KEY_KEY);
    log.info("Reading X or Y from {}", yKey);

    Y = new LongObjectMap<float[]>();

    Iterable<MatrixRow> in;
    try {
        in = new AvroFileSource<MatrixRow>(Namespaces.toPath(yKey), (AvroType<MatrixRow>) ptype).read(conf);
    } catch (IOException e) {
        throw new IllegalStateException(e);
    }

    RealMatrix theYTY = null;
    int dimension = 0;
    long count = 0;
    for (MatrixRow record : in) {
        long keyID = record.getRowId();
        float[] vector = record.getValues();
        Preconditions.checkNotNull(vector, "Vector was null for %s?", keyID);

        if (theYTY == null) {
            dimension = vector.length;
            theYTY = new Array2DRowRealMatrix(dimension, dimension);
        }
        for (int row = 0; row < dimension; row++) {
            double rowValue = vector[row];
            for (int col = 0; col < dimension; col++) {
                theYTY.addToEntry(row, col, rowValue * vector[col]);
            }
        }

        if (expectedIDs == null || expectedIDs.contains(keyID)) {
            Y.put(keyID, vector);
        }

        if (++count % 1000 == 0) {
            context.progress();
        }
    }

    Preconditions.checkNotNull(theYTY);
    YTY = theYTY;
}

---

From source file:com.cloudera.oryx.als.computation.recommend.RecommendReduceFn.java

License:Open Source License

@Override
public void initialize() {
    super.initialize();
    Configuration conf = getConfiguration();
    String yKey = conf.get(RecommendStep.Y_KEY_KEY);
    try {//  w  ww .j a  v  a  2 s. c o m
        partialY = ComputationDataUtils.loadPartialY(getPartition(), getNumPartitions(), yKey, conf);
    } catch (IOException e) {
        throw new CrunchRuntimeException(e);
    }
    numRecs = ConfigUtils.getDefaultConfig().getInt("model.recommend.how-many");
    Preconditions.checkArgument(numRecs > 0, "# recs must be positive: %s", numRecs);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• »