Example usage for org.apache.hadoop.conf Configuration setStrings

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration setStrings.

Prototype

public void setStrings(String name, String... values) 

Source Link

Document

Set the array of string values for the name property as as comma delimited values.

Usage

From source file:com.bah.culvert.hive.CulvertHiveUtils.java

License:Apache License

/**
 * Set the hive column names in the conf. Corresponds 1-1 to the mappings.
 * /*  w  w w .  j  a  v  a 2s .co  m*/
 * @param conf The conf to set the hive columns in.
 * @param names The column names.
 */
public static void setHiveColumnNamesInConf(Configuration conf, String[] names) {
    conf.setStrings(CULVERT_HIVE_COLUMN_NAMES_CONF_KEY, names);
}

---

From source file:com.bah.culvert.hive.CulvertHiveUtils.java

License:Apache License

/**
 * Set the hive column types in the conf. Corresponds 1-1 to the mappings.
 * //  www  .j  av  a2  s . c  om
 * @param conf The configuration to set the types in.
 * @param types The types to set.
 */
public static void setHiveColumnTypesInConf(Configuration conf, String[] types) {
    conf.setStrings(CULVERT_HIVE_COLUMN_TYPES_CONF_KEY, types);
}

---

From source file:com.basho.riak.hadoop.config.RiakConfig.java

License:Apache License

/**
 * Set the {@link KeyLister} implementation to use.
 * /*from www  .jav a 2s . c  o m*/
 * @param conf
 *            the {@link Configuration} to update
 * @param lister
 *            the {@link KeyLister} to use
 * @return the configuration updated with a serialized version of the lister
 *         provided
 */
public static <T extends KeyLister> Configuration setKeyLister(Configuration conf, T lister)
        throws IOException {
    conf.setClass(KEY_LISTER_CLASS_PROPERTY, lister.getClass(), KeyLister.class);
    conf.setStrings(KEY_LISTER_INIT_STRING_PROPERTY, lister.getInitString());
    return conf;
}

---

From source file:com.ci.backports.avro.mapreduce.AvroJob.java

License:Apache License

private static void addAvroSerialization(Configuration conf) {
    Collection<String> serializations = conf.getStringCollection("io.serializations");
    if (!serializations.contains(AvroSerialization.class.getName())) {
        serializations.add(AvroSerialization.class.getName());
        conf.setStrings("io.serializations", serializations.toArray(new String[0]));
    }/*from ww  w.j  a  v a 2 s  .com*/
}

---

From source file:com.cloudera.recordservice.mapreduce.MapReduceTest.java

License:Apache License

private void verifyInputSplitsTable(int numSplits, int numCols, String tbl, String... cols) throws IOException {
    Configuration config = new Configuration();
    config.set(ConfVars.TBL_NAME_CONF.name, tbl);
    if (cols.length > 0) {
        config.setStrings(ConfVars.COL_NAMES_CONF.name, cols);
    }/*from   ww  w.j a v  a 2  s  .com*/
    verifyInputSplits(numSplits, numCols, config);
}

---

From source file:com.cloudera.recordservice.mapreduce.MapReduceTest.java

License:Apache License

@Test
public void testGetSplits() throws IOException {
    Configuration config = new Configuration();

    boolean exceptionThrown = false;
    try {//from  w  ww . ja  va  2 s. c o  m
        PlanUtil.getSplits(config, new Credentials());
    } catch (IllegalArgumentException e) {
        exceptionThrown = true;
        assertTrue(e.getMessage().contains("No input specified"));
    }
    assertTrue(exceptionThrown);

    // Set db/table and make sure it works.
    config.set(ConfVars.TBL_NAME_CONF.name, "tpch.nation");
    PlanUtil.getSplits(config, new Credentials());

    // Also set input. This should fail.
    config.set(FileInputFormat.INPUT_DIR, "/test");
    exceptionThrown = false;
    try {
        PlanUtil.getSplits(config, new Credentials());
    } catch (IllegalArgumentException e) {
        exceptionThrown = true;
        assertTrue(e.getMessage(), e.getMessage().contains("More than one input specified"));
    }
    assertTrue(exceptionThrown);

    // Unset the table and set columns. INPUT_DIR and columns don't work now.
    config.unset(ConfVars.TBL_NAME_CONF.name);
    config.setStrings(ConfVars.COL_NAMES_CONF.name, "a");
    exceptionThrown = false;
    try {
        PlanUtil.getSplits(config, new Credentials());
    } catch (IllegalArgumentException e) {
        exceptionThrown = true;
        assertTrue(e.getMessage().contains("Column projections can only be specified with table inputs."));
    }
    assertTrue(exceptionThrown);

    // Test some cases that work
    verifyInputSplitsTable(1, 4, "tpch.nation");
    verifyInputSplitsTable(2, 12, "rs.alltypes");
    verifyInputSplitsTable(1, 1, "tpch.nation", "n_name");
    verifyInputSplitsTable(2, 3, "rs.alltypes", "int_col", "double_col", "string_col");
    verifyInputSplitsPath(1, 1, "/test-warehouse/tpch.nation");

    // Test some cases using the config utility.
    config.clear();
    RecordServiceConfig.setInputTable(config, null, "tpch.nation", "n_nationkey", "n_comment");
    verifyInputSplits(1, 2, config);

    exceptionThrown = false;
    try {
        verifyInputSplitsTable(1, 1, "tpch.nation", "bad");
    } catch (IOException e) {
        exceptionThrown = true;
        assertTrue(e.getCause() instanceof RecordServiceException);
        RecordServiceException ex = (RecordServiceException) e.getCause();
        assertEquals(RecordServiceException.ErrorCode.INVALID_REQUEST, ex.code);
    }
    assertTrue(exceptionThrown);

    exceptionThrown = false;
    try {
        verifyInputSplitsPath(1, 1, "/test-warehouse/tpch.nation,/test-warehouse/tpch.nation");
    } catch (IllegalArgumentException e) {
        exceptionThrown = true;
        assertTrue(e.getMessage().contains("Only reading a single directory is currently supported."));
    }
    assertTrue(exceptionThrown);
}

---

From source file:com.cloudera.recordservice.mr.RecordServiceConfig.java

License:Apache License

/**
 * Sets the input configuration to read 'cols' from 'db.tbl'. If the tbl is fully
 * qualified, db should be null./*from  w w  w.  j  a v  a2s. co  m*/
 * If cols is empty, all cols in the table are read.
 */
public static void setInputTable(Configuration conf, String db, String tbl, String... cols) {
    if (tbl == null || tbl == "") {
        throw new IllegalArgumentException("'tbl' must be non-empty");
    }
    if (db != null && db != "")
        tbl = db + "." + tbl;
    conf.set(ConfVars.TBL_NAME_CONF.name, tbl);
    if (cols != null && cols.length > 0) {
        for (int i = 0; i < cols.length; ++i) {
            if (cols[i] == null || cols[i].isEmpty()) {
                throw new IllegalArgumentException("Column list cannot contain empty names.");
            }
        }
        conf.setStrings(ConfVars.COL_NAMES_CONF.name, cols);
    }
}

---

From source file:com.cloudera.sqoop.mapreduce.MySQLDumpImportJob.java

License:Apache License

/**
 * Configure the inputformat to use for the job.
 *//*  w ww  .j ava 2  s .  c  om*/
protected void configureInputFormat(Job job, String tableName, String tableClassName, String splitByCol)
        throws ClassNotFoundException, IOException {

    if (null == tableName) {
        LOG.error("mysqldump-based import cannot support free-form query imports.");
        LOG.error("Do not use --direct and --query together for MySQL.");
        throw new IOException("null tableName for MySQLDumpImportJob.");
    }

    ConnManager mgr = getContext().getConnManager();
    String username = options.getUsername();
    if (null == username || username.length() == 0) {
        DBConfiguration.configureDB(job.getConfiguration(), mgr.getDriverClass(), options.getConnectString());
    } else {
        DBConfiguration.configureDB(job.getConfiguration(), mgr.getDriverClass(), options.getConnectString(),
                username, options.getPassword());
    }

    String[] colNames = options.getColumns();
    if (null == colNames) {
        colNames = mgr.getColumnNames(tableName);
    }

    String[] sqlColNames = null;
    if (null != colNames) {
        sqlColNames = new String[colNames.length];
        for (int i = 0; i < colNames.length; i++) {
            sqlColNames[i] = mgr.escapeColName(colNames[i]);
        }
    }

    // It's ok if the where clause is null in DBInputFormat.setInput.
    String whereClause = options.getWhereClause();

    // We can't set the class properly in here, because we may not have the
    // jar loaded in this JVM. So we start by calling setInput() with
    // DBWritable and then overriding the string manually.

    // Note that mysqldump also does *not* want a quoted table name.
    DataDrivenDBInputFormat.setInput(job, DBWritable.class, tableName, whereClause,
            mgr.escapeColName(splitByCol), sqlColNames);

    Configuration conf = job.getConfiguration();
    conf.setInt(MySQLUtils.OUTPUT_FIELD_DELIM_KEY, options.getOutputFieldDelim());
    conf.setInt(MySQLUtils.OUTPUT_RECORD_DELIM_KEY, options.getOutputRecordDelim());
    conf.setInt(MySQLUtils.OUTPUT_ENCLOSED_BY_KEY, options.getOutputEnclosedBy());
    conf.setInt(MySQLUtils.OUTPUT_ESCAPED_BY_KEY, options.getOutputEscapedBy());
    conf.setBoolean(MySQLUtils.OUTPUT_ENCLOSE_REQUIRED_KEY, options.isOutputEncloseRequired());
    String[] extraArgs = options.getExtraArgs();
    if (null != extraArgs) {
        conf.setStrings(MySQLUtils.EXTRA_ARGS_KEY, extraArgs);
    }

    LOG.debug("Using InputFormat: " + inputFormatClass);
    job.setInputFormatClass(getInputFormatClass());
}

---

From source file:com.cloudera.sqoop.mapreduce.MySQLExportJob.java

License:Apache License

@Override
/**/*from   w w w .  j a  va2  s. co m*/
 * Configure the inputformat to use for the job.
 */
protected void configureInputFormat(Job job, String tableName, String tableClassName, String splitByCol)
        throws ClassNotFoundException, IOException {

    // Configure the delimiters, etc.
    Configuration conf = job.getConfiguration();
    conf.setInt(MySQLUtils.OUTPUT_FIELD_DELIM_KEY, options.getOutputFieldDelim());
    conf.setInt(MySQLUtils.OUTPUT_RECORD_DELIM_KEY, options.getOutputRecordDelim());
    conf.setInt(MySQLUtils.OUTPUT_ENCLOSED_BY_KEY, options.getOutputEnclosedBy());
    conf.setInt(MySQLUtils.OUTPUT_ESCAPED_BY_KEY, options.getOutputEscapedBy());
    conf.setBoolean(MySQLUtils.OUTPUT_ENCLOSE_REQUIRED_KEY, options.isOutputEncloseRequired());
    String[] extraArgs = options.getExtraArgs();
    if (null != extraArgs) {
        conf.setStrings(MySQLUtils.EXTRA_ARGS_KEY, extraArgs);
    }

    ConnManager mgr = context.getConnManager();
    String username = options.getUsername();
    if (null == username || username.length() == 0) {
        DBConfiguration.configureDB(job.getConfiguration(), mgr.getDriverClass(), options.getConnectString());
    } else {
        DBConfiguration.configureDB(job.getConfiguration(), mgr.getDriverClass(), options.getConnectString(),
                username, options.getPassword());
    }

    String[] colNames = options.getColumns();
    if (null == colNames) {
        colNames = mgr.getColumnNames(tableName);
    }

    String[] sqlColNames = null;
    if (null != colNames) {
        sqlColNames = new String[colNames.length];
        for (int i = 0; i < colNames.length; i++) {
            sqlColNames[i] = mgr.escapeColName(colNames[i]);
        }
    }

    // Note that mysqldump also does *not* want a quoted table name.
    DataDrivenDBInputFormat.setInput(job, DBWritable.class, tableName, null, null, sqlColNames);

    // Configure the actual InputFormat to use. 
    super.configureInputFormat(job, tableName, tableClassName, splitByCol);
}

---

From source file:com.datasalt.pangool.tuplemr.serialization.TupleSerialization.java

License:Apache License

/**
 * Use this method to enable this serialization in Hadoop
 */// ww w .ja  v a 2 s.  c  o  m
public static void enableSerialization(Configuration conf) {
    String serClass = TupleSerialization.class.getName();
    Collection<String> currentSers = conf.getStringCollection("io.serializations");

    if (currentSers.size() == 0) {
        conf.set("io.serializations", serClass);
        return;
    }

    // Check if it is already present
    if (!currentSers.contains(serClass)) {
        currentSers.add(serClass);
        conf.setStrings("io.serializations", currentSers.toArray(new String[] {}));
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• »