Example usage for org.apache.hadoop.conf Configuration getStrings

List of usage examples for org.apache.hadoop.conf Configuration getStrings

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. getStrings

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getStrings.

Prototype

public String[] getStrings(String name, String... defaultValue)

Source Link

Document

Get the comma delimited values of the name property as an array of Strings.

Usage

From source file:com.alibaba.wasp.zookeeper.ZKConfig.java

License:Apache License

/**
 * Make a Properties object holding ZooKeeper config equivalent to zoo.cfg.
 * If there is a zoo.cfg in the classpath, simply read it in. Otherwise parse
 * the corresponding config options from the Wasp XML configs and generate
 * the appropriate ZooKeeper properties.
 * @param conf Configuration to read from.
 * @return Properties holding mappings representing ZooKeeper zoo.cfg file.
 *///  w ww .  j ava2 s  . c o m
public static Properties makeZKProps(Configuration conf) {
    // First check if there is a zoo.cfg in the CLASSPATH. If so, simply read
    // it and grab its configuration properties.
    ClassLoader cl = FQuorumPeer.class.getClassLoader();
    final InputStream inputStream = cl.getResourceAsStream(FConstants.ZOOKEEPER_CONFIG_NAME);
    if (inputStream != null) {
        try {
            return parseZooCfg(conf, inputStream);
        } catch (IOException e) {
            LOG.warn("Cannot read " + FConstants.ZOOKEEPER_CONFIG_NAME + ", loading from XML files", e);
        }
    }

    // Otherwise, use the configuration options from Wasp's XML files.
    Properties zkProperties = new Properties();

    // Directly map all of the wasp.zookeeper.property.KEY properties.
    for (Entry<String, String> entry : conf) {
        String key = entry.getKey();
        if (key.startsWith(FConstants.ZK_CFG_PROPERTY_PREFIX)) {
            String zkKey = key.substring(FConstants.ZK_CFG_PROPERTY_PREFIX_LEN);
            String value = entry.getValue();
            // If the value has variables substitutions, need to do a get.
            if (value.contains(VARIABLE_START)) {
                value = conf.get(key);
            }
            zkProperties.put(zkKey, value);
        }
    }

    // If clientPort is not set, assign the default.
    if (zkProperties.getProperty(FConstants.CLIENT_PORT_STR) == null) {
        zkProperties.put(FConstants.CLIENT_PORT_STR, FConstants.DEFAULT_ZOOKEPER_CLIENT_PORT);
    }

    // Create the server.X properties.
    int peerPort = conf.getInt("wasp.zookeeper.peerport", 2888);
    int leaderPort = conf.getInt("wasp.zookeeper.leaderport", 3888);

    final String[] serverHosts = conf.getStrings(FConstants.ZOOKEEPER_QUORUM, FConstants.LOCALHOST);
    for (int i = 0; i < serverHosts.length; ++i) {
        String serverHost = serverHosts[i];
        String address = serverHost + ":" + peerPort + ":" + leaderPort;
        String key = "server." + i;
        zkProperties.put(key, address);
    }

    return zkProperties;
}

From source file:com.digitalpebble.behemoth.gate.GATEAnnotationFilters.java

License:Apache License

public static GATEAnnotationFilters getFilters(Configuration config) {
    GATEAnnotationFilters filter = new GATEAnnotationFilters();

    filter.annotationSetName = config.get("gate.annotationset.output", "");
    String[] stypes = config.getStrings("gate.annotations.filter", "");
    String[] sFeatFilt = config.getStrings("gate.features.filter", "");
    String[] docFeatFilt = config.getStrings("gate.docfeatures.filter", "");

    // the featurefilters have the following form : Type.featureName
    filter.featfilts = new HashMap<String, Set<String>>();
    for (String ff : sFeatFilt) {
        String[] fp = ff.split("\\.");
        if (fp.length != 2)
            continue;
        Set<String> fnames = filter.featfilts.get(fp[0]);
        if (fnames == null) {
            fnames = new HashSet<String>();
            filter.featfilts.put(fp[0], fnames);
        }/*from w  ww .j a  va 2  s  .  c  o  m*/
        fnames.add(fp[1]);
    }

    filter.types = new HashSet<String>();
    for (String s : stypes) {
        s = s.trim();
        if (s.length() > 0)
            filter.types.add(s);
    }

    filter.docFeatureNames = new HashSet<String>();
    for (String s : docFeatFilt) {
        s = s.trim();
        if (s.length() > 0)
            filter.docFeatureNames.add(s);
    }

    return filter;
}

From source file:com.facebook.hiveio.conf.StrConfOption.java

License:Apache License

/**
 * Get array of values for key// ww  w  . j  a va 2s .  c o m
 * @param conf Configuration
 * @return array of values for key
 */
public String[] getArray(Configuration conf) {
    return conf.getStrings(getKey(), defaultValue);
}

From source file:com.github.hdl.tensorflow.yarn.app.TFAmContainer.java

License:Apache License

public Map<String, String> setJavaEnv(Configuration conf) {
    Map<String, String> env = new HashMap<String, String>();

    // Add AppMaster.jar location to classpath
    // At some point we should not be required to add
    // the hadoop specific classpaths to the env.
    // It should be provided out of the box.
    // For now setting all required classpaths including
    // the classpath to "." for the application jar
    StringBuilder classPathEnv = new StringBuilder(ApplicationConstants.Environment.CLASSPATH.$$())
            .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*");
    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) {
        classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR);
        classPathEnv.append(c.trim());//from  w  w  w .j a  v a  2  s  . c  om
    }
    classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./log4j.properties");

    // add the runtime classpath needed for tests to work
    if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
        classPathEnv.append(':');
        classPathEnv.append(System.getProperty("java.class.path"));
    }

    env.put("CLASSPATH", classPathEnv.toString());
    return env;
}

From source file:com.github.hdl.tensorflow.yarn.app.TFContainer.java

License:Apache License

public Map<String, String> setJavaEnv(Configuration conf, String tfServerJar) {
    // Set the java environment
    Map<String, String> env = new HashMap<String, String>();

    // Add TFServerLauncher.jar location to classpath
    StringBuilder classPathEnv = new StringBuilder(ApplicationConstants.Environment.CLASSPATH.$$())
            .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*");

    // Add hadoop's jar location to classpath
    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) {
        classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR);
        classPathEnv.append(c.trim());// ww  w. jav a2  s  . com
    }
    classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./log4j.properties");

    // add the runtime classpath needed for tests to work
    if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
        classPathEnv.append(':');
        classPathEnv.append(System.getProperty("java.class.path"));
    }

    if (tfServerJar != null) {
        classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR);
        classPathEnv.append(tfServerJar);
    }
    env.put("CLASSPATH", classPathEnv.toString());
    return env;
}

From source file:com.google.cloud.bigtable.mapreduce.Export.java

License:Apache License

private static Scan getConfiguredScanForJob(Configuration conf, String[] args) throws IOException {
    Scan s = new Scan();
    // Optional arguments.
    // Set Scan Versions
    int versions = args.length > 2 ? Integer.parseInt(args[2]) : 1;
    s.setMaxVersions(versions);/*from  ww  w.  j  ava 2 s  . co  m*/
    // Set Scan Range
    long startTime = args.length > 3 ? Long.parseLong(args[3]) : 0L;
    long endTime = args.length > 4 ? Long.parseLong(args[4]) : Long.MAX_VALUE;
    s.setTimeRange(startTime, endTime);
    // Set cache blocks
    s.setCacheBlocks(false);
    // set Start and Stop row
    if (conf.get(TableInputFormat.SCAN_ROW_START) != null) {
        s.setStartRow(Bytes.toBytes(conf.get(TableInputFormat.SCAN_ROW_START)));
    }
    if (conf.get(TableInputFormat.SCAN_ROW_STOP) != null) {
        s.setStopRow(Bytes.toBytes(conf.get(TableInputFormat.SCAN_ROW_STOP)));
    }
    // Set Scan Column Family
    boolean raw = Boolean.parseBoolean(conf.get(RAW_SCAN));
    if (raw) {
        s.setRaw(raw);
    }

    if (conf.get(TableInputFormat.SCAN_COLUMN_FAMILY) != null) {
        s.addFamily(Bytes.toBytes(conf.get(TableInputFormat.SCAN_COLUMN_FAMILY)));
    }
    // Add additional comma-separated families
    for (String family : conf.getStrings(SCAN_COLUMN_FAMILIES, new String[0])) {
        s.addFamily(Bytes.toBytes(family));
    }
    // Set RowFilter or Prefix Filter if applicable.
    Filter exportFilter = getExportFilter(args);
    if (exportFilter != null) {
        LOG.info("Setting Scan Filter for Export.");
        s.setFilter(exportFilter);
    }

    int batching = conf.getInt(EXPORT_BATCHING, -1);
    if (batching != -1) {
        try {
            s.setBatch(batching);
        } catch (IncompatibleFilterException e) {
            LOG.error("Batching could not be set", e);
        }
    }
    LOG.info("versions=" + versions + ", starttime=" + startTime + ", endtime=" + endTime
            + ", keepDeletedCells=" + raw);
    return s;
}

From source file:eu.stratosphere.yarn.Utils.java

License:Apache License

public static void setupEnv(Configuration conf, Map<String, String> appMasterEnv) {
    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), c.trim());
    }/*from  ww w  .  ja v  a2s  .com*/
    addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), Environment.PWD.$() + File.separator + "*");
}

From source file:gobblin.yarn.YarnHelixUtils.java

License:Apache License

/**
 * Get environment variables in a {@link java.util.Map} used when launching a Yarn container.
 *
 * @param yarnConfiguration a Hadoop {@link Configuration} object carrying Hadoop/Yarn configuration properties
 * @return a {@link java.util.Map} storing environment variables used when launching a Yarn container
 *//*from  w  ww . j  a v a2  s . co  m*/
@SuppressWarnings("deprecation")
public static Map<String, String> getEnvironmentVariables(Configuration yarnConfiguration) {
    Map<String, String> environmentVariableMap = Maps.newHashMap();

    if (System.getenv().containsKey(ApplicationConstants.Environment.JAVA_HOME.key())) {
        Apps.addToEnvironment(environmentVariableMap, ApplicationConstants.Environment.JAVA_HOME.key(),
                System.getenv(ApplicationConstants.Environment.JAVA_HOME.key()));
    }

    // Add jars/files in the working directory of the ApplicationMaster to the CLASSPATH
    Apps.addToEnvironment(environmentVariableMap, ApplicationConstants.Environment.CLASSPATH.key(),
            ApplicationConstants.Environment.PWD.$());
    Apps.addToEnvironment(environmentVariableMap, ApplicationConstants.Environment.CLASSPATH.key(),
            ApplicationConstants.Environment.PWD.$() + File.separator + "*");

    String[] classpaths = yarnConfiguration.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH);
    if (classpaths != null) {
        for (String classpath : classpaths) {
            Apps.addToEnvironment(environmentVariableMap, ApplicationConstants.Environment.CLASSPATH.key(),
                    classpath.trim());
        }
    }

    return environmentVariableMap;
}

From source file:hws.core.ContainerUtils.java

License:Apache License

public static void setupContainerEnv(Map<String, String> containerEnv, Configuration conf) {
    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        Apps.addToEnvironment(containerEnv, Environment.CLASSPATH.name(), c.trim());
    }//from  w w w.j  a  va 2 s .c  om
    Apps.addToEnvironment(containerEnv, Environment.CLASSPATH.name(),
            Environment.PWD.$() + File.separator + "*");
}

From source file:InvertedIndex.NLineRecordReader.java

License:Apache License

public boolean isIgnoreBadCompress(Configuration job, Throwable exception) {
    if (null != job && job.getBoolean("mapred.ignore.badcompress", false)) {
        String exceptionStr = StringUtils.stringifyException(exception);
        String[] keywordsBL = job.getStrings("mapred.ignore.badcompress.keywords.blacklist",
                "Could not obtain block");
        if (null != keywordsBL) {
            for (String keyword : keywordsBL) {
                if (null != keyword && exceptionStr.contains(keyword)) {
                    return false;
                }//from w  ww.  ja  va  2 s .c o  m
            }
        }

        String[] keywords = job.getStrings("mapred.ignore.badcompress.keywords",
                "org.apache.hadoop.io.compress.DecompressorStream",
                "org.apache.hadoop.io.compress.MultiMemberGZIPInputStream",
                "org.apache.hadoop.io.compress.GzipCodec$GzipInputStream",
                "com.hadoop.compression.lzo.LzopCodec$LzopInputStream");

        if (null != keywords) {
            for (String keyword : keywords) {
                if (null != keyword && exceptionStr.contains(keyword)) {
                    return true;
                }
            }
        }
    }
    return false;
}