Example usage for org.apache.hadoop.conf Configuration getStrings

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getStrings.

Prototype

public String[] getStrings(String name, String... defaultValue) 

Source Link

Document

Get the comma delimited values of the name property as an array of Strings.

Usage

From source file:ml.shifu.guagua.yarn.util.YarnUtils.java

License:Apache License

/**
 * Populate the environment string map to be added to the environment vars in a remote execution container.
 * //from   w w w .  j  a  va2  s  . c o m
 * @param env
 *            the map of env var values.
 * @param conf
 *            the Configuration to pull values from.
 */
public static void addLocalClasspathToEnv(final Map<String, String> env, final Configuration conf) {
    StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$());

    // add current folder
    classPathEnv.append(File.pathSeparatorChar).append("./*");

    // add yarn app classpath
    for (String cpEntry : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        classPathEnv.append(File.pathSeparatorChar).append(cpEntry.trim());
    }

    for (String jar : conf.getStrings(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH,
            org.apache.hadoop.util.StringUtils
                    .getStrings(MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH))) {
        classPathEnv.append(File.pathSeparatorChar).append(jar.trim());
    }

    // add the runtime classpath needed for tests to work
    if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
        classPathEnv.append(File.pathSeparatorChar).append(Environment.CLASSPATH.$());
    }

    // add guagua app jar file
    String path = getFileName(conf.get(GuaguaYarnConstants.GUAGUA_YARN_APP_JAR));
    classPathEnv.append(GuaguaYarnConstants.CURRENT_DIR).append(path).append(File.pathSeparatorChar);

    // Any libraries we may have copied over?
    String libs = conf.get(GuaguaYarnConstants.GUAGUA_YARN_APP_LIB_JAR);
    if (StringUtils.isNotEmpty(libs)) {
        for (String jar : Splitter.on(GuaguaYarnConstants.GUAGUA_APP_LIBS_SEPERATOR).split(libs)) {
            classPathEnv.append(GuaguaYarnConstants.CURRENT_DIR).append(getFileName(jar.trim()))
                    .append(File.pathSeparatorChar);
        }
    }

    // add log4j
    classPathEnv.append(GuaguaYarnConstants.CURRENT_DIR).append(GuaguaYarnConstants.GUAGUA_LOG4J_PROPERTIES)
            .append(File.pathSeparatorChar);

    // add guagua-conf.xml
    classPathEnv.append(GuaguaYarnConstants.CURRENT_DIR).append(GuaguaYarnConstants.GUAGUA_CONF_FILE);

    env.put(Environment.CLASSPATH.toString(), classPathEnv.toString());
}

---

From source file:mvm.rya.indexing.accumulo.ConfigUtils.java

License:Apache License

private static Set<URI> getPredicates(Configuration conf, String confName) {
    String[] validPredicateStrings = conf.getStrings(confName, new String[] {});
    Set<URI> predicates = new HashSet<URI>();
    for (String prediateString : validPredicateStrings) {
        predicates.add(new URIImpl(prediateString));
    }/*  ww  w .  j av  a 2  s.  com*/
    return predicates;
}

---

From source file:net.shun.mapreduce.lib.input.XmlRecordReader.java

License:Apache License

public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException {
    FileSplit split = (FileSplit) genericSplit;
    Configuration job = context.getConfiguration();
    this.maxLineLength = job.getInt("mapred.linerecordreader.maxlength", Integer.MAX_VALUE);
    String[] beginMarks = job.getStrings("mapred.xmlrecordreader.begin", "<page>");
    this.beginMark = beginMarks[0];
    String[] endMarks = job.getStrings("mapred.xmlrecordreader.begin", "</page>");
    this.endMark = endMarks[0];

    start = split.getStart();//from w w w  .  j  a v a2 s.  c o  m
    end = start + split.getLength();
    final Path file = split.getPath();
    compressionCodecs = new CompressionCodecFactory(job);
    final CompressionCodec codec = compressionCodecs.getCodec(file);

    // open the file and seek to the start of the split
    FileSystem fs = file.getFileSystem(job);
    FSDataInputStream fileIn = fs.open(split.getPath());
    fileIn.seek(start);
    in = new BufferedInputStream(fileIn);
    /*
    boolean skipFirstLine = false;
    if (codec != null) {
      in = new LineReader(codec.createInputStream(fileIn), job);
      end = Long.MAX_VALUE;
    } else {
      if (start != 0) {
        skipFirstLine = true;
        --start;
        fileIn.seek(start);
      }
      in = new LineReader(fileIn, job);
    }
    if (skipFirstLine) {  // skip first line and re-establish "start".
      start += in.readLine(new Text(), 0,
                   (int)Math.min((long)Integer.MAX_VALUE, end - start));
    }
    */
    this.pos = start;
    readUntilMatch(beginMark, false, null);
}

---

From source file:org.apache.gora.hbase.util.HBaseFilterUtil.java

License:Apache License

public HBaseFilterUtil(Configuration conf) throws GoraException {
    String[] factoryClassNames = conf.getStrings("gora.hbase.filter.factories",
            "org.apache.gora.hbase.util.DefaultFactory");

    for (String factoryClass : factoryClassNames) {
        try {/*from  w ww .j a v a 2s  .  c  om*/
            @SuppressWarnings("unchecked")
            FilterFactory<K, T> factory = (FilterFactory<K, T>) ReflectionUtils.newInstance(factoryClass);
            for (String filterClass : factory.getSupportedFilters()) {
                factories.put(filterClass, factory);
            }
            factory.setHBaseFitlerUtil(this);
        } catch (Exception e) {
            throw new GoraException(e);
        }
    }
}

---

From source file:org.apache.gora.mongodb.filters.MongoFilterUtil.java

License:Apache License

public MongoFilterUtil(final Configuration conf) throws GoraException {
    String[] factoryClassNames = conf.getStrings(MONGO_FILTER_FACTORIES_PARAMETER,
            MONGO_FILTERS_DEFAULT_FACTORY);

    for (String factoryClass : factoryClassNames) {
        try {//from www. ja  va2s.c  o m
            FilterFactory<K, T> factory = (FilterFactory<K, T>) ReflectionUtils.newInstance(factoryClass);
            for (String filterClass : factory.getSupportedFilters()) {
                factories.put(filterClass, factory);
            }
            factory.setFilterUtil(this);
        } catch (Exception e) {
            throw new GoraException(e);
        }
    }
}

---

From source file:org.apache.hama.zookeeper.QuorumPeer.java

License:Apache License

/**
 * Make a Properties object holding ZooKeeper config equivalent to zoo.cfg. If
 * there is a zoo.cfg in the classpath, simply read it in. Otherwise parse the
 * corresponding config options from the Hama XML configs and generate the
 * appropriate ZooKeeper properties./*from   w w w  . java 2  s. c o  m*/
 * 
 * @param conf Configuration to read from.
 * @return Properties holding mappings representing ZooKeeper zoo.cfg file.
 */
public static Properties makeZKProps(Configuration conf) {
    // First check if there is a zoo.cfg in the CLASSPATH. If so, simply read
    // it and grab its configuration properties.
    ClassLoader cl = QuorumPeer.class.getClassLoader();
    InputStream inputStream = cl.getResourceAsStream(ZOOKEEPER_CONFIG_NAME);
    if (inputStream != null) {
        try {
            return parseZooCfg(conf, inputStream);
        } catch (IOException e) {
            LOG.warn("Cannot read " + ZOOKEEPER_CONFIG_NAME + ", loading from XML files", e);
        }
    }

    // Otherwise, use the configuration options from Hama's XML files.
    Properties zkProperties = new Properties();

    // Set the max session timeout from the provided client-side timeout
    zkProperties.setProperty("maxSessionTimeout", conf.get(Constants.ZOOKEEPER_SESSION_TIMEOUT, "1200000"));

    // Directly map all of the hama.zookeeper.property.KEY properties.
    for (Entry<String, String> entry : conf) {
        String key = entry.getKey();
        if (key.startsWith(ZK_CFG_PROPERTY)) {
            String zkKey = key.substring(ZK_CFG_PROPERTY_SIZE);
            String value = entry.getValue();
            // If the value has variables substitutions, need to do a get.
            if (value.contains(VARIABLE_START)) {
                value = conf.get(key);
            }
            zkProperties.put(zkKey, value);
        }
    }

    // If clientPort is not set, assign the default
    if (zkProperties.getProperty(ZOOKEEPER_CLIENT_PORT) == null) {
        zkProperties.put(ZOOKEEPER_CLIENT_PORT, DEFAULT_ZOOKEEPER_CLIENT_PORT);
    }

    // Create the server.X properties.
    int peerPort = conf.getInt("hama.zookeeper.peerport", 2888);
    int leaderPort = conf.getInt("hama.zookeeper.leaderport", 3888);

    String[] serverHosts = conf.getStrings(ZOOKEEPER_QUORUM, "localhost");
    for (int i = 0; i < serverHosts.length; ++i) {
        String serverHost = serverHosts[i];
        String address = serverHost + ":" + peerPort + ":" + leaderPort;
        String key = "server." + i;
        zkProperties.put(key, address);
    }

    return zkProperties;
}

---

From source file:org.apache.hoya.tools.HoyaUtils.java

License:Apache License

/**
 * Build up the classpath for execution/*www. ja va 2 s.c  o  m*/
 * -behaves very differently on a mini test cluster vs a production
 * production one.
 *
 * @param hoyaConfDir relative path to the dir containing hoya config options to put on the
 *          classpath -or null
 * @param libdir directory containing the JAR files
 * @param config the configuration
 * @param usingMiniMRCluster flag to indicate the MiniMR cluster is in use
 * (and hence the current classpath should be used, not anything built up)
 * @return a classpath
 */
public static String buildClasspath(String hoyaConfDir, String libdir, Configuration config,
        boolean usingMiniMRCluster) {
    // Add AppMaster.jar location to classpath
    // At some point we should not be required to add
    // the hadoop specific classpaths to the env.
    // It should be provided out of the box.
    // For now setting all required classpaths including
    // the classpath to "." for the application jar
    StringBuilder classPathEnv = new StringBuilder();
    // add the runtime classpath needed for tests to work
    if (usingMiniMRCluster) {
        // for mini cluster we pass down the java CP properties
        // and nothing else
        classPathEnv.append(System.getProperty("java.class.path"));
    } else {
        char col = File.pathSeparatorChar;
        classPathEnv.append(ApplicationConstants.Environment.CLASSPATH.$());
        String[] strs = config.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
                YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH);
        if (strs != null) {
            for (String c : strs) {
                classPathEnv.append(col);
                classPathEnv.append(c.trim());
            }
        }
        classPathEnv.append(col).append("./").append(libdir).append("/*");
        if (hoyaConfDir != null) {
            classPathEnv.append(col).append(hoyaConfDir);
        }
    }
    return classPathEnv.toString();
}

---

From source file:org.apache.nutch.parse.metatags.MetaTagsParser.java

License:Apache License

public void setConf(Configuration conf) {
    this.conf = conf;
    // specify whether we want a specific subset of metadata
    // by default take everything we can find
    String[] values = conf.getStrings("metatags.names", "*");
    for (String val : values) {
        metatagset.add(val.toLowerCase(Locale.ROOT));
    }/*from w w  w.j a va  2s.  com*/
}

---

From source file:org.apache.nutch.parse.replace.ReplaceParser.java

License:Apache License

public void setConf(Configuration conf) {
    this.conf = conf;
    String[] values = conf.getStrings("parse.replace.regexp", null);
    if (values != null) {
        this.parseConf(values);
    }//  w  w  w.j a  va  2s .com
}

---

From source file:org.apache.nutch.scoring.similarity.cosine.Model.java

License:Apache License

/**
 * Retrieves mingram and maxgram from configuration
 * @param conf Configuration to retrieve mingram and maxgram
 * @return ngram array as mingram at first index and maxgram at second index
   *///w  w  w .  ja  va2s . c om
public static int[] retrieveNgrams(Configuration conf) {
    int[] ngramArr = new int[2];
    //Check if user has specified mingram or ngram for ngram cosine model
    String[] ngramStr = conf.getStrings("scoring.similarity.ngrams", "1,1");
    //mingram
    ngramArr[0] = Integer.parseInt(ngramStr[0]);
    if (ngramStr.length > 1) {
        //maxgram
        ngramArr[1] = Integer.parseInt(ngramStr[1]);
    } else {
        //maxgram
        ngramArr[1] = ngramArr[0];
    }
    return ngramArr;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• »