Example usage for org.apache.hadoop.conf Configuration get

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration get.

Prototype

public String get(String name) 

Source Link

Document

Get the value of the name property, null if no such property exists.

Usage

From source file:co.nubetech.hiho.job.DBQueryInputJob.java

License:Apache License

public void populateHiveConfigurationForMultiplePartition(Configuration conf) throws IOException {

    String columnName = null, columnType = null;
    ArrayList columnsValues = new ArrayList();

    ArrayList query = new ArrayList();
    ArrayList table = new ArrayList();

    String partitionByData = conf.get(HIHOConf.HIVE_PARTITION_BY);
    boolean isInputQueryDelimited = false;
    String queries = null;/*from  w w  w . j  a  v  a2s  .c o  m*/
    if (conf.get(DBConfiguration.INPUT_QUERY) != null) {
        isInputQueryDelimited = true;
        queries = conf.get(DBConfiguration.INPUT_QUERY);
    }
    // /
    StringTokenizer partitionByTokens = new StringTokenizer(partitionByData, ":");
    columnName = partitionByTokens.nextToken();
    columnType = partitionByTokens.nextToken();
    StringTokenizer partitionByValues = new StringTokenizer(partitionByTokens.nextToken(), ",");
    int counter = 0;
    int tokenCounts = partitionByValues.countTokens();
    while (partitionByValues.hasMoreTokens()) {
        columnsValues.add(counter, partitionByValues.nextToken());
        counter++;
    }
    // /
    if (isInputQueryDelimited) {
        StringTokenizer queryTokens = new StringTokenizer(queries, ";");
        counter = 0;
        while (queryTokens.hasMoreTokens()) {
            query.add(counter, queryTokens.nextToken());
            counter++;
        }
    } else {
        StringTokenizer tableTokens = new StringTokenizer(conf.get(DBConfiguration.INPUT_TABLE_NAME_PROPERTY),
                ";");
        counter = 0;
        while (tableTokens.hasMoreTokens()) {
            table.add(counter, tableTokens.nextToken());
            counter++;
        }

    }
    for (int jobCounter = 0; jobCounter < tokenCounts; jobCounter++) {
        if (isInputQueryDelimited) {
            ;
            conf.set(DBConfiguration.INPUT_QUERY, query.get(jobCounter).toString());
        } else {
            conf.set(DBConfiguration.INPUT_TABLE_NAME_PROPERTY, table.get(jobCounter).toString());
        }
        conf.set(HIHOConf.INPUT_OUTPUT_PATH, conf.get(HIHOConf.INPUT_OUTPUT_PATH) + jobCounter);
        // conf.set(HIHOConf.HIVE_TABLE_OVERWRITE, "true");
        String partitionBy = columnName + ":" + columnType + ":" + columnsValues.get(jobCounter).toString();
        conf.set(HIHOConf.HIVE_PARTITION_BY, partitionBy);
        runJobs(conf, jobCounter);

    }

}

---

From source file:co.nubetech.hiho.job.DBQueryInputJob.java

License:Apache License

public void runJobs(Configuration conf, int jobCounter) throws IOException {

    try {/* w  w  w  .  j  a v a2s  .  c o m*/
        checkMandatoryConfs(conf);
    } catch (HIHOException e1) {
        e1.printStackTrace();
        throw new IOException(e1);
    }

    Job job = new Job(conf);
    for (Entry<String, String> entry : conf) {
        logger.warn("key, value " + entry.getKey() + "=" + entry.getValue());
    }

    // logger.debug("Number of maps " +
    // conf.getInt("mapred.map.tasks", 1));
    // conf.setInt(JobContext.NUM_MAPS,
    // conf.getInt("mapreduce.job.maps", 1));
    // job.getConfiguration().setInt("mapred.map.tasks", 4);
    job.getConfiguration().setInt(MRJobConfig.NUM_MAPS, conf.getInt(HIHOConf.NUMBER_MAPPERS, 1));
    logger.warn("Number of maps " + conf.getInt(MRJobConfig.NUM_MAPS, 1));

    job.setJobName("Import job");
    job.setJarByClass(DBQueryInputJob.class);

    String strategy = conf.get(HIHOConf.INPUT_OUTPUT_STRATEGY);
    OutputStrategyEnum os = OutputStrategyEnum.value(strategy);
    if (os == null) {
        throw new IllegalArgumentException("Wrong value of output strategy. Please correct");
    }
    if (os != OutputStrategyEnum.AVRO) {
        switch (os) {

        case DUMP: {
            // job.setMapperClass(DBImportMapper.class);
            break;
        }
        /*
         * case AVRO: { job.setMapperClass(DBInputAvroMapper.class); //
         * need avro in cp // job.setJarByClass(Schema.class); // need
         * jackson which is needed by avro - ugly! //
         * job.setJarByClass(ObjectMapper.class);
         * job.setMapOutputKeyClass(NullWritable.class);
         * job.setMapOutputValueClass(AvroValue.class);
         * job.setOutputKeyClass(NullWritable.class);
         * job.setOutputValueClass(AvroValue.class);
         * job.setOutputFormatClass(AvroOutputFormat.class);
         * 
         * AvroOutputFormat.setOutputPath(job, new
         * Path(getConf().get(HIHOConf.INPUT_OUTPUT_PATH))); break; }
         */
        case DELIMITED: {
            job.setMapperClass(DBInputDelimMapper.class);
            job.setMapOutputKeyClass(Text.class);
            job.setMapOutputValueClass(Text.class);
            job.setOutputKeyClass(Text.class);
            job.setOutputValueClass(Text.class);
            job.setOutputFormatClass(NoKeyOnlyValueOutputFormat.class);

            NoKeyOnlyValueOutputFormat.setOutputPath(job, new Path(getConf().get(HIHOConf.INPUT_OUTPUT_PATH)));
        }
        case JSON: {
            // job.setMapperClass(DBImportJsonMapper.class);
            // job.setJarByClass(ObjectMapper.class);
            break;
        }
        default: {
            job.setMapperClass(DBInputDelimMapper.class);
            job.setMapOutputKeyClass(Text.class);
            job.setMapOutputValueClass(Text.class);
            job.setOutputKeyClass(Text.class);
            job.setOutputValueClass(Text.class);
            job.setOutputFormatClass(NoKeyOnlyValueOutputFormat.class);

            NoKeyOnlyValueOutputFormat.setOutputPath(job, new Path(getConf().get(HIHOConf.INPUT_OUTPUT_PATH)));
            break;
        }
        }

        String inputQuery = conf.get(DBConfiguration.INPUT_QUERY);
        String inputBoundingQuery = conf.get(DBConfiguration.INPUT_BOUNDING_QUERY);
        logger.debug("About to set the params");
        DBQueryInputFormat.setInput(job, inputQuery, inputBoundingQuery, params);
        logger.debug("Set the params");

        job.setNumReduceTasks(0);

        try {
            // job.setJarByClass(Class.forName(conf.get(
            // org.apache.hadoop.mapred.lib.db.DBConfiguration.DRIVER_CLASS_PROPERTY)));
            logger.debug("OUTPUT format class is " + job.getOutputFormatClass());

            /*
             * org.apache.hadoop.mapreduce.OutputFormat<?, ?> output =
             * ReflectionUtils.newInstance(job.getOutputFormatClass(),
             * job.getConfiguration()); output.checkOutputSpecs(job);
             */
            logger.debug("Class is " + ReflectionUtils
                    .newInstance(job.getOutputFormatClass(), job.getConfiguration()).getClass().getName());
            job.waitForCompletion(false);
            if (conf.get(HIHOConf.INPUT_OUTPUT_LOADTO) != null) {
                generateHiveScript(conf, job, jobCounter);
                generatePigScript(conf, job);
            }

        }
        /*
         * catch (HIHOException h) { h.printStackTrace(); }
         */
        catch (Exception e) {
            e.printStackTrace();
        } catch (HIHOException e) {
            e.printStackTrace();
        }
    }
    // avro to be handled differently, thanks to all the incompatibilities
    // in the apis.
    else {
        String inputQuery = conf.get(DBConfiguration.INPUT_QUERY);
        String inputBoundingQuery = conf.get(DBConfiguration.INPUT_BOUNDING_QUERY);
        logger.debug("About to set the params");
        // co.nubetech.apache.hadoop.mapred.DBQueryInputFormat.setInput(job,
        // inputQuery, inputBoundingQuery, params);
        logger.debug("Set the params");

        JobConf jobConf = new JobConf(conf);

        try {
            GenericDBWritable queryWritable = getDBWritable(jobConf);
            Schema pair = DBMapper.getPairSchema(queryWritable.getColumns());

            AvroJob.setMapOutputSchema(jobConf, pair);
            GenericRecordAvroOutputFormat.setOutputPath(jobConf,
                    new Path(getConf().get(HIHOConf.INPUT_OUTPUT_PATH)));

            co.nubetech.apache.hadoop.mapred.DBQueryInputFormat.setInput(jobConf, inputQuery,
                    inputBoundingQuery, params);
            jobConf.setInputFormat(co.nubetech.apache.hadoop.mapred.DBQueryInputFormat.class);
            jobConf.setMapperClass(DBInputAvroMapper.class);
            jobConf.setMapOutputKeyClass(NullWritable.class);
            jobConf.setMapOutputValueClass(AvroValue.class);
            jobConf.setOutputKeyClass(NullWritable.class);
            jobConf.setOutputValueClass(Text.class);
            jobConf.setOutputFormat(GenericRecordAvroOutputFormat.class);
            jobConf.setJarByClass(DBQueryInputJob.class);
            jobConf.setStrings("io.serializations",
                    "org.apache.hadoop.io.serializer.JavaSerialization,org.apache.hadoop.io.serializer.WritableSerialization,org.apache.avro.mapred.AvroSerialization");
            jobConf.setNumReduceTasks(0);
            /*
             * jobConf.setOutputFormat(org.apache.hadoop.mapred.
             * SequenceFileOutputFormat.class);
             * org.apache.hadoop.mapred.SequenceFileOutputFormat
             * .setOutputPath(jobConf, new
             * Path(getConf().get(HIHOConf.INPUT_OUTPUT_PATH)));
             */
            JobClient.runJob(jobConf);
        } catch (Throwable e) {
            e.printStackTrace();
        }

    }

}

---

From source file:co.nubetech.hiho.job.ExportToDB.java

License:Apache License

public void checkMandatoryConfs(Configuration conf) throws HIHOException {
    if (conf.get(DBConfiguration.DRIVER_CLASS_PROPERTY) == null) {
        throw new HIHOException("JDBC driver configuration is not specified,please specify JDBC driver class.");
    }// w w  w .ja v a2 s. c o  m
    if (conf.get(DBConfiguration.URL_PROPERTY) == null) {
        throw new HIHOException("JDBC url path configuration is empty,please specify JDBC url path.");
    }
    if (!conf.get(DBConfiguration.DRIVER_CLASS_PROPERTY).contains("hsqldb")) {
        if (conf.get(DBConfiguration.USERNAME_PROPERTY) == null) {
            throw new HIHOException("JDBC user name configuration is empty,please specify JDBC user name.");
        }
        if (conf.get(DBConfiguration.PASSWORD_PROPERTY) == null) {
            throw new HIHOException("JDBC password configuration is empty,please specify JDBC password.");
        }
    }
    if (conf.get(HIHOConf.INPUT_OUTPUT_DELIMITER) == null) {
        throw new HIHOException("The provided delimiter is empty, please specify delimiter.");
    }
    if (conf.get(HIHOConf.NUMBER_MAPPERS) == null) {
        throw new HIHOException("The provided number of mappers is empty, please specify number of mappers.");
    }
    if (inputPath == null) {
        throw new HIHOException("The provided input path is empty, please specify inputPath.");
    }
    if (tableName == null) {
        throw new HIHOException("The provided table name is empty, please specify tableName.");
    }
    if (columnNames == null) {
        throw new HIHOException("The provided column name is empty, please specify columnName.");
    }
}

---

From source file:co.nubetech.hiho.job.ExportToFTPServer.java

License:Apache License

public void checkMandatoryConfs(Configuration conf) throws HIHOException {
    if (inputPath == null) {
        throw new HIHOException("The provided inputPath is empty, please specify inputPath");
    }//w w w. j av a 2  s  .  c om
    if (outputPath == null) {
        throw new HIHOException("The outputPath is not defined, please specify outputPath");
    }
    if (conf.get(HIHOConf.FTP_USER) == null) {
        throw new HIHOException("The FTP UserName is not defined, please specify FTP UserName");
    }
    if (conf.get(HIHOConf.FTP_ADDRESS) == null) {
        throw new HIHOException("The FTP Address is not defined, please specify FTP Address");
    }
    if (conf.get(HIHOConf.FTP_PORT) == null) {
        throw new HIHOException("The FTP Port Number is not defined, please specify FTP Port Number");
    }
    if (conf.get(HIHOConf.FTP_PASSWORD) == null) {
        throw new HIHOException("The FTP Password is not defined, please specify FTP Password");
    }
}

---

From source file:co.nubetech.hiho.job.ExportToMySQLDB.java

License:Apache License

public void checkMandatoryConfs(Configuration conf) throws HIHOException {
    if (inputPath == null) {
        throw new HIHOException("The provided inputPath is empty, please specify inputPath");
    }/* ww  w . j a  v a 2  s  .  c o  m*/
    if (conf.get(DBConfiguration.URL_PROPERTY) == null) {
        throw new HIHOException("The JDBC URL is not specified, please specify JDBC URL");
    }
    if (conf.get(DBConfiguration.USERNAME_PROPERTY) == null) {
        throw new HIHOException("The JDBC USERNAME is not specified, please specify JDBC USERNAME");
    }
    if (conf.get(DBConfiguration.PASSWORD_PROPERTY) == null) {
        throw new HIHOException("The JDBC PASSWORD is not defined, please specify JDBC PASSWORD");
    }
    if (conf.get(HIHOConf.LOAD_QUERY_SUFFIX) == null) {
        throw new HIHOException("The Suffix for query is not defined, please specify suffix");
    }
}

---

From source file:co.nubetech.hiho.job.ExportToMySQLDB.java

License:Apache License

@Override
public int run(String[] args) throws IOException {

    Configuration conf = getConf();
    populateConfiguration(args, conf);//  ww w . ja  va 2 s .c  om
    try {
        checkMandatoryConfs(conf);
    } catch (HIHOException e1) {
        e1.printStackTrace();
        throw new IOException(e1);
    }
    Job job = new Job(conf);
    job.setJobName("MySQLBulkLoading");
    job.setMapperClass(MySQLLoadDataMapper.class);
    job.setJarByClass(MySQLLoadDataMapper.class);
    for (Entry<String, String> entry : conf) {
        logger.debug("key, value " + entry.getKey() + "=" + entry.getValue());
    }
    // verify required properties are loaded
    logger.debug(conf.get(DBConfiguration.URL_PROPERTY));
    logger.debug(conf.get(DBConfiguration.USERNAME_PROPERTY));
    logger.debug(conf.get(DBConfiguration.PASSWORD_PROPERTY));

    job.setNumReduceTasks(0);
    job.setInputFormatClass(FileStreamInputFormat.class);
    FileStreamInputFormat.addInputPath(job, new Path(inputPath));
    job.setMapOutputKeyClass(NullWritable.class);
    job.setMapOutputValueClass(NullWritable.class);
    // job.setJarByClass(com.mysql.jdbc.Driver.class);
    job.setOutputFormatClass(NullOutputFormat.class);

    int ret = 0;
    try {
        ret = job.waitForCompletion(true) ? 0 : 1;
    } catch (Exception e) {
        e.printStackTrace();
    }
    return ret;
}

---

From source file:co.nubetech.hiho.job.ExportToOracleDb.java

License:Apache License

public void checkMandatoryConfs(Configuration conf) throws HIHOException {
    if (inputPath == null) {
        throw new HIHOException("The provided inputPath is empty, please specify inputPath");
    }//from www  . j  a v  a  2 s.  c o m
    if (conf.get(HIHOConf.ORACLE_FTP_USER) == null) {
        throw new HIHOException("The Oracle FTP UserName is not specified, please specify Oracle FTP UserName");
    }
    if (conf.get(HIHOConf.ORACLE_FTP_ADDRESS) == null) {
        throw new HIHOException("The Oracle FTP Address is not specified, please specify Oracle FTP Address");
    }
    if (conf.get(HIHOConf.ORACLE_FTP_PORT) == null) {
        throw new HIHOException(
                "The Oracle FTP Port Number is not defined, please specify Oracle FTP Port Number");
    }
    if (conf.get(HIHOConf.ORACLE_FTP_PASSWORD) == null) {
        throw new HIHOException("The Oracle FTP Password is not defined, please specify Oracle FTP Password");
    }
    if (conf.get(HIHOConf.ORACLE_EXTERNAL_TABLE_DIR) == null) {
        throw new HIHOException(
                "The Oracle External Table Directory is not specified, please specify Oracle External Table Directory");
    }
    if (conf.get(DBConfiguration.DRIVER_CLASS_PROPERTY) == null) {
        throw new HIHOException("The JDBC Driver is not specified, please specify JDBC Driver");
    }
    if (conf.get(DBConfiguration.URL_PROPERTY) == null) {
        throw new HIHOException("The JDBC URL is not specified, please specify JDBC URL");
    }
    if (conf.get(DBConfiguration.USERNAME_PROPERTY) == null) {
        throw new HIHOException("The JDBC USERNAME is not specified, please specify JDBC USERNAME");
    }
    if (conf.get(DBConfiguration.PASSWORD_PROPERTY) == null) {
        throw new HIHOException("The JDBC PASSWORD is not specified, please specify JDBC PASSWORD");
    }
    if (conf.get(HIHOConf.EXTERNAL_TABLE_DML) == null) {
        throw new HIHOException(
                "The query to create external table is not specified, please specify the create query for external table");
    }

}

---

From source file:co.nubetech.hiho.job.ExportToOracleDb.java

License:Apache License

@Override
public int run(String[] args) throws IOException {
    Configuration conf = getConf();
    for (Entry<String, String> entry : conf) {
        logger.debug("key, value " + entry.getKey() + "=" + entry.getValue());
    }//from   ww  w. jav a 2  s . com

    for (int i = 0; i < args.length; i++) {
        logger.debug("Remaining arguments are" + " " + args[i]);
    }
    populateConfiguration(args, conf);

    try {
        checkMandatoryConfs(conf);
    } catch (HIHOException e1) {
        e1.printStackTrace();
        throw new IOException(e1);
    }

    Job job = new Job(conf);
    job.setJobName("OracleLoading");
    job.setMapperClass(OracleLoadMapper.class);
    job.setJarByClass(ExportToOracleDb.class);
    job.getConfiguration().setInt(MRJobConfig.NUM_MAPS, conf.getInt(HIHOConf.NUMBER_MAPPERS, 1));

    try {
        // we first create the external table definition
        String query = conf.get(HIHOConf.EXTERNAL_TABLE_DML);
        // create table if user has specified
        if (query != null) {
            this.runQuery(query, conf);
        }
    } catch (HIHOException e1) {

        e1.printStackTrace();
    }

    // verify required properties are loaded

    job.setNumReduceTasks(0);
    job.setInputFormatClass(FileStreamInputFormat.class);
    FileStreamInputFormat.addInputPath(job, new Path(inputPath));
    job.setMapOutputKeyClass(NullWritable.class);
    job.setMapOutputValueClass(NullWritable.class);
    // job.setJarByClass(com.mysql.jdbc.Driver.class);
    job.setOutputFormatClass(NullOutputFormat.class);

    int ret = 0;
    try {

        ret = job.waitForCompletion(true) ? 0 : 1;
    } catch (Exception e) {
        e.printStackTrace();
    }
    // run alter table query and add locations
    try {
        this.runQuery(getAlterTableDML(new Path(inputPath), conf), conf);
    } catch (HIHOException e1) {

        e1.printStackTrace();
    }
    return ret;
}

---

From source file:co.nubetech.hiho.job.ExportToOracleDb.java

License:Apache License

public static String getAlterTableDML(Path inputPath, Configuration conf) throws IOException, HIHOException {
    // after running the job, we need to alter the external table to take
    // care of the files we have added
    FileStatus[] contents = inputPath.getFileSystem(conf).listStatus(inputPath);
    if (contents != null) {
        StringBuilder dml = new StringBuilder();
        dml.append(" ALTER TABLE ");
        dml.append(getTableName(conf.get(HIHOConf.EXTERNAL_TABLE_DML)));
        dml.append(" LOCATION (");
        int i = 0;
        for (FileStatus content : contents) {
            String fileName = content.getPath().getName();
            dml.append("\'");
            dml.append(fileName);// w ww.  j  av a  2  s . co  m
            dml.append("\'");
            // add comma uptil one and last
            if (i < contents.length - 1) {
                dml.append(",");
            }
            i++;
        }
        // execute dml to alter location

        dml.append(")");
        return dml.toString();
    }
    return null;
}

---

From source file:co.nubetech.hiho.job.ExportToOracleDb.java

License:Apache License

/**
 * This function creates the external table. the following queries have been
 * tested: create table age( i Number, n Varchar(20), a Number )
 * organization external ( type oracle_loader default directory ext_dir
 * access parameters ( records delimited by newline fields terminated by ','
 * missing field values are null ) location('file.txt') ) reject limit
 * unlimited//  w w w .  j  a  v  a2  s.co  m
 * 
 * 
 * The following queries are bound to fail create table ( i Number, n
 * Varchar(20), a Number ) organization external ( type oracle_loader
 * default directory ext_dir access parameters ( records delimited by
 * newline fields terminated by ',' missing field values are null )
 * location( 'file.txt' ) ) reject limit unlimited
 * 
 * @param query
 * @param conf
 * @throws HIHOException
 */
public void runQuery(String query, Configuration conf) throws HIHOException {
    try {
        String driver = conf.get(DBConfiguration.DRIVER_CLASS_PROPERTY);
        String url = conf.get(DBConfiguration.URL_PROPERTY);
        String usr = conf.get(DBConfiguration.USERNAME_PROPERTY);
        String pswd = conf.get(DBConfiguration.PASSWORD_PROPERTY);

        Class.forName(driver);
        Connection conn = DriverManager.getConnection(url, usr, pswd);

        Statement stmt = conn.createStatement();

        stmt.executeQuery(query);

        stmt.close();
        conn.close();

    } catch (Exception e) {
        e.printStackTrace();
        throw new HIHOException("Sql syntax error in query " + query + e);
    }

}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• »