Example usage for org.apache.hadoop.conf Configuration get

List of usage examples for org.apache.hadoop.conf Configuration get

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. get

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration get.

Prototype

public String get(String name)

Source Link

Document

Get the value of the name property, null if no such property exists.

Usage

From source file:com.aerospike.hadoop.mapreduce.AerospikeConfigUtil.java

License:Apache License

public static String getOutputBinName(Configuration conf) {
    String binname = conf.get(OUTPUT_BINNAME);
    log.info("using " + OUTPUT_BINNAME + " = " + binname);
    return binname;
}

From source file:com.aerospike.hadoop.mapreduce.AerospikeConfigUtil.java

License:Apache License

public static String getOutputKeyName(Configuration conf) {
    String keyname = conf.get(OUTPUT_KEYNAME);
    log.info("using " + OUTPUT_KEYNAME + " = " + keyname);
    return keyname;
}

From source file:com.ailk.oci.ocnosql.tools.load.csvbulkload.CsvBulkLoadTool.java

License:Apache License

@Override
public int run(String[] args) throws Exception {

    HBaseConfiguration.addHbaseResources(getConf());
    Configuration conf = getConf();
    String quorum = conf.get("hbase.zookeeper.quorum");
    String clientPort = conf.get("hbase.zookeeper.property.clientPort");
    LOG.info("hbase.zookeeper.quorum=" + quorum);
    LOG.info("hbase.zookeeper.property.clientPort=" + clientPort);
    LOG.info("phoenix.query.dateFormat=" + conf.get("phoenix.query.dateFormat"));

    CommandLine cmdLine = null;/*from   w  w  w  .j  a v  a  2  s . co m*/
    try {
        cmdLine = parseOptions(args);
        LOG.info("JdbcUrl=" + getJdbcUrl(quorum + ":" + clientPort));
    } catch (IllegalStateException e) {
        printHelpAndExit(e.getMessage(), getOptions());
    }
    Class.forName(DriverManager.class.getName());
    Connection conn = DriverManager.getConnection(getJdbcUrl(quorum + ":" + clientPort));
    String tableName = cmdLine.getOptionValue(TABLE_NAME_OPT.getOpt());
    String schemaName = cmdLine.getOptionValue(SCHEMA_NAME_OPT.getOpt());
    String qualifiedTableName = getQualifiedTableName(schemaName, tableName);
    List<ColumnInfo> importColumns = buildImportColumns(conn, cmdLine, qualifiedTableName);

    LOG.info("tableName=" + tableName);
    LOG.info("schemaName=" + schemaName);
    LOG.info("qualifiedTableName=" + qualifiedTableName);

    configureOptions(cmdLine, importColumns, getConf());

    try {
        validateTable(conn, schemaName, tableName);
    } finally {
        conn.close();
    }

    Path inputPath = new Path(cmdLine.getOptionValue(INPUT_PATH_OPT.getOpt()));
    Path outputPath = null;
    if (cmdLine.hasOption(OUTPUT_PATH_OPT.getOpt())) {
        outputPath = new Path(cmdLine.getOptionValue(OUTPUT_PATH_OPT.getOpt()));
    } else {
        outputPath = new Path("/tmp/" + UUID.randomUUID());
    }
    LOG.info("Configuring HFile output path to {}", outputPath);

    Job job = new Job(getConf(),
            "Phoenix MapReduce import for " + getConf().get(PhoenixCsvToKeyValueMapper.TABLE_NAME_CONFKEY));

    // Allow overriding the job jar setting by using a -D system property at startup
    if (job.getJar() == null) {
        job.setJarByClass(PhoenixCsvToKeyValueMapper.class);
    }
    job.setInputFormatClass(TextInputFormat.class);
    FileInputFormat.addInputPath(job, inputPath);

    FileSystem.get(getConf());
    FileOutputFormat.setOutputPath(job, outputPath);

    job.setMapperClass(PhoenixCsvToKeyValueMapper.class);
    job.setMapOutputKeyClass(ImmutableBytesWritable.class);
    job.setMapOutputValueClass(KeyValue.class);

    HTable htable = new HTable(getConf(), qualifiedTableName);

    // Auto configure partitioner and reducer according to the Main Data table
    HFileOutputFormat.configureIncrementalLoad(job, htable);

    LOG.info("Running MapReduce import job from {} to {}", inputPath, outputPath);
    boolean success = job.waitForCompletion(true);
    if (!success) {
        LOG.error("Import job failed, check JobTracker for details");
        return 1;
    }

    LOG.info("Loading HFiles from {}", outputPath);
    LoadIncrementalHFiles loader = new LoadIncrementalHFiles(getConf());
    loader.doBulkLoad(outputPath, htable);
    htable.close();

    LOG.info("Incremental load complete");

    LOG.info("Removing output directory {}", outputPath);
    if (!FileSystem.get(getConf()).delete(outputPath, true)) {
        LOG.error("Removing output directory {} failed", outputPath);
    }

    return 0;
}

From source file:com.ailk.oci.ocnosql.tools.load.csvbulkload.PhoenixCsvToKeyValueMapper.java

License:Apache License

@Override
protected void setup(Context context) throws IOException, InterruptedException {

    Configuration conf = context.getConfiguration();
    String jdbcUrl = getJdbcUrl(conf);

    // This statement also ensures that the driver class is loaded
    LOG.info("Connection with driver {} with url {}", PhoenixDriver.class.getName(), jdbcUrl);

    try {/*from   ww  w .j av a  2  s. c om*/
        conn = (PhoenixConnection) DriverManager.getConnection(jdbcUrl);
    } catch (SQLException e) {
        throw new RuntimeException(e);
    }

    upsertListener = new MapperUpsertListener(context, conf.getBoolean(IGNORE_INVALID_ROW_CONFKEY, true));
    csvUpsertExecutor = buildUpsertExecutor(conf);
    csvLineParser = new CsvLineParser(conf.get(FIELD_DELIMITER_CONFKEY).charAt(0));

    preUpdateProcessor = loadPreUpsertProcessor(conf);

    // ?
    // ?
    List<String> importColumnList = new ArrayList<String>();
    for (ColumnInfo colInfo : buildColumnInfoList(conf)) {
        importColumnList.add(colInfo.getColumnName());
    }
    // ?hash?
    List<String> rowPrefixColumns = Lists
            .newArrayList(Splitter.on(",").trimResults().split(conf.get(ROW_PREFIX_COLUMNS)));
    // rowkey?hash?
    rowPrefixColIdxs = new ArrayList<Integer>();
    for (String rpCol : rowPrefixColumns) {
        // ?1csv??
        rowPrefixColIdxs.add(importColumnList.indexOf(rpCol) - 1);
    }

    // rowkey
    List<String> rowColumns = Lists.newArrayList(Splitter.on(",").trimResults().split(conf.get(ROW_COLUMNS)));
    rowColIdxs = new ArrayList<Integer>();
    for (String rCol : rowColumns) {
        // ?1csv??
        rowColIdxs.add(importColumnList.indexOf(rCol) - 1);
    }

    // 
    List<String> uniqueIndexColumns = Lists
            .newArrayList(Splitter.on(",").trimResults().split(conf.get(UNIQUE_INDEX_COLUMNS, "_allColumns")));
    if (uniqueIndexColumns.size() == 1 && uniqueIndexColumns.get(0).equals("_allColumns")) {
        unqIdxColIdxs = null;
    } else {
        unqIdxColIdxs = new ArrayList<Integer>();
        for (String rCol : uniqueIndexColumns) {
            // ?1csv??
            unqIdxColIdxs.add(importColumnList.indexOf(rCol) - 1);
        }
    }

    // ?rowkey???(md5)
    rowKeyGenerator = buildRowKeyGenerator(conf.get(ROW_PREFIX_ALG, "md5"));
    separator = conf.get(FIELD_DELIMITER_CONFKEY);

    // ??
    rowGentemp = new StringBuilder();
}

From source file:com.ailk.oci.ocnosql.tools.load.csvbulkload.PhoenixCsvToKeyValueMapper.java

License:Apache License

/**
 * Build up the JDBC URL for connecting to Phoenix.
 * //from   w w w . j  ava2 s  .  co m
 * @return the full JDBC URL for a Phoenix connection
 */
@VisibleForTesting
static String getJdbcUrl(Configuration conf) {
    String zkQuorum = conf.get("hbase.zookeeper.quorum") + ":"
            + conf.get("hbase.zookeeper.property.clientPort");
    if (zkQuorum == null) {
        throw new IllegalStateException(HConstants.ZOOKEEPER_QUORUM + " is not configured");
    }
    return PhoenixRuntime.JDBC_PROTOCOL + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkQuorum;
}

From source file:com.ailk.oci.ocnosql.tools.load.csvbulkload.PhoenixCsvToKeyValueMapper.java

License:Apache License

@VisibleForTesting
CsvUpsertExecutor buildUpsertExecutor(Configuration conf) {
    String tableName = conf.get(TABLE_NAME_CONFKEY);
    String arraySeparator = conf.get(ARRAY_DELIMITER_CONFKEY, CSVCommonsLoader.DEFAULT_ARRAY_ELEMENT_SEPARATOR);
    Preconditions.checkNotNull(tableName, "table name is not configured");

    List<ColumnInfo> columnInfoList = buildColumnInfoList(conf);

    return CsvUpsertExecutor.create(conn, tableName, columnInfoList, upsertListener, arraySeparator);
}

From source file:com.ailk.oci.ocnosql.tools.load.csvbulkload.PhoenixCsvToKeyValueMapper.java

License:Apache License

/**
 * Build the list of ColumnInfos for the import based on information in the
 * configuration./*from   w ww  .ja  va2s .  c om*/
 */
@VisibleForTesting
static List<ColumnInfo> buildColumnInfoList(Configuration conf) {
    return Lists.newArrayList(Iterables.transform(Splitter.on("|").split(conf.get(COLUMN_INFO_CONFKEY)),
            new Function<String, ColumnInfo>() {
                @Nullable
                @Override
                public ColumnInfo apply(@Nullable String input) {
                    if (input.isEmpty()) {
                        // An empty string represents a null that was passed
                        // in to
                        // the configuration, which corresponds to an input
                        // column
                        // which is to be skipped
                        return null;
                    }
                    return ColumnInfo.fromString(input);
                }
            }));
}

From source file:com.ailk.oci.ocnosql.tools.load.mutiple.MutipleColumnImporterMapper.java

License:Apache License

/**
* Handles initializing this class with objects specific to it (i.e., the parser).
* Common initialization that might be leveraged by a subsclass is done in
* <code>doSetup</code>. Hence a subclass may choose to override this method
* and call <code>doSetup</code> as well before handling it's own custom params.
*
* @param context/*  w  ww  . j a  v a2  s. c  o m*/
*/
@Override
protected void setup(Context context) {
    doSetup(context);
    Configuration conf = context.getConfiguration();
    String columnsDefi = conf.get(CommonConstants.COLUMNS);
    parser = new MutipleColumnImportTsv.TsvParser(columnsDefi, separator);
    String notNeedLoadColumns = conf.get(CommonConstants.NOTNEEDLOADCOLUMNS);
    if (!StringUtils.isEmpty(notNeedLoadColumns)) {
        String[] notNeedLoadColumnArr = notNeedLoadColumns.split(",");
        for (int i = 0; i < notNeedLoadColumnArr.length; i++) {
            if (notNeedLoadColumnArr[i].contains("F:")) {
                notNeedLoadColumnQulifiers.add(notNeedLoadColumnArr[i].substring(2));
            } else {
                notNeedLoadColumnQulifiers.add(notNeedLoadColumnArr[i]);
            }
        }
    }
}

From source file:com.ailk.oci.ocnosql.tools.load.mutiple.MutipleColumnImporterMapper.java

License:Apache License

/**
 * Handles common parameter initialization that a subclass might want to leverage.
 * @param context/* w ww  . j a va2s.  co m*/
 */
protected void doSetup(Context context) {
    Configuration conf = context.getConfiguration();

    // If a custom separator has been used,
    // decode it back from Base64 encoding.
    separator = conf.get(CommonConstants.SEPARATOR);
    if (separator == null) {
        separator = CommonConstants.DEFAULT_SEPARATOR;
    } else {
        separator = new String(Base64.decode(separator));
    }

    //    ts = conf.getLong(ImportTsv.TIMESTAMP_CONF_KEY, System.nanoTime());//currentTimeMillis());

    skipBadLines = context.getConfiguration().getBoolean(CommonConstants.SKIPBADLINE, true);
    badLineCount = context.getCounter("ImportTsv", "Bad Lines");
    totalLineCount = context.getCounter("ImportTsv", "total Lines");
    /* //rowkey? tableRowKeyGenerator ??
    String rowkeyGennerator = context.getConfiguration().get(CommonConstants.ROWKEY_GENERATOR);
    if(RowKeyGeneratorHolder.TYPE.md5.name().equalsIgnoreCase(rowkeyGennerator)){
       rowkeyGenerator = new MD5RowKeyGenerator();
    }
    */
    tableName = conf.get(CommonConstants.TABLE_NAME);
    List<GenRKStep> genRKStepList = TableConfiguration.getInstance().getTableGenRKSteps(tableName, conf);
    rowkeyGenerator = new TableRowKeyGenerator(conf, genRKStepList);

    index = 0;
}

From source file:com.ailk.oci.ocnosql.tools.load.mutiple.MutipleColumnImportTsv.java

License:Apache License

/**
 * Sets up the actual job.//from   w  w  w  .j av  a2s .c o  m
 *
 * @param conf  The current configuration.
 * @return The newly created job.
 * @throws IOException When setting up the job fails.
 */
public static Job createSubmittableJob(Configuration conf, String tableName, String inputPath,
        String tmpOutputPath) throws IOException, ClassNotFoundException {

    // Support non-XML supported characters
    // by re-encoding the passed separator as a Base64 string.
    String actualSeparator = conf.get(CommonConstants.SEPARATOR);
    if (actualSeparator != null) {
        conf.set(CommonConstants.SEPARATOR, Base64.encodeBytes(actualSeparator.getBytes()));
    }
    String tableNameConf = conf.get(CommonConstants.TABLE_NAME);
    if (tableNameConf == null) {
        conf.set(CommonConstants.TABLE_NAME, tableName);
    }

    // See if a non-default Mapper was set
    String mapperClassName = conf.get(MAPPER_CONF_KEY);
    Class mapperClass = mapperClassName != null ? Class.forName(mapperClassName) : DEFAULT_MAPPER;

    Path inputDir = new Path(inputPath);
    Job job = new Job(conf, NAME + "_" + tableName);
    job.setJarByClass(MutipleColumnImportTsv.class);
    FileInputFormat.setInputPaths(job, inputDir);

    //??Dimporttsv.inputFormatInputFormat,TextInputFormat
    String inputFmtName = conf.get(CommonConstants.INPUTFORMAT,
            "org.apache.hadoop.mapreduce.lib.input.TextInputFormat");
    LOG.info(CommonConstants.INPUTFORMAT + " is " + inputFmtName);
    Class<? extends InputFormat> inputFmtClass = Class.forName(inputFmtName).asSubclass(InputFormat.class);
    job.setInputFormatClass(inputFmtClass);
    job.setMapperClass(mapperClass);

    String hfileOutPath = tmpOutputPath;
    if (hfileOutPath != null) {
        if (!doesTableExist(tableName)) {
            createTable(conf, tableName);
        }
        HTable table = new HTable(conf, tableName);
        //      job.setReducerClass(MutipleColumnReducer.class);
        Path outputDir = new Path(hfileOutPath);
        FileOutputFormat.setOutputPath(job, outputDir);
        job.setMapOutputKeyClass(ImmutableBytesWritable.class);
        job.setMapOutputValueClass(Put.class);
        HFileOutputFormat.configureIncrementalLoad(job, table);
    } else {
        // No reducers.  Just write straight to table.  Call initTableReducerJob
        // to set up the TableOutputFormat.
        TableMapReduceUtil.initTableReducerJob(tableName, null, job);
        job.setNumReduceTasks(0);
    }

    TableMapReduceUtil.addDependencyJars(job);
    TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
            com.google.common.base.Function.class /* Guava used by TsvParser */);
    return job;
}