Example usage for org.apache.hadoop.conf Configuration set

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration set.

Prototype

public void set(String name, String value) 

Source Link

Document

Set the value of the name property.

Usage

From source file:co.nubetech.hiho.hive.TestHiveUtility.java

License:Apache License

@Test
public void testGetTmpCreateQuery() throws HIHOException {
    ColumnInfo intColumn = new ColumnInfo(0, Types.INTEGER, "intColumn");
    ColumnInfo stringColumn = new ColumnInfo(1, Types.VARCHAR, "stringColumn");
    ArrayList<ColumnInfo> columns = new ArrayList<ColumnInfo>();
    columns.add(intColumn);/*  ww  w. j a  v a  2 s  .c  om*/
    columns.add(stringColumn);
    Configuration conf = new Configuration();
    conf.set(DBConfiguration.INPUT_TABLE_NAME_PROPERTY, "employee");
    GenericDBWritable writable = new GenericDBWritable(columns, null);

    conf.set(HIHOConf.INPUT_OUTPUT_STRATEGY, "DELIMITED");
    conf.set(HIHOConf.INPUT_OUTPUT_DELIMITER, ",");
    assertEquals(
            "CREATE TABLE `employeetmp` ( `intColumn` int,`stringColumn` string) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE",
            HiveUtility.getTmpCreateQuery(conf, writable));
}

---

From source file:co.nubetech.hiho.hive.TestHiveUtility.java

License:Apache License

@Test
public void testGetColumnsForPartitionedCreateTables() {
    Configuration conf = new Configuration();
    conf.set(HIHOConf.HIVE_PARTITION_BY, "country:string:us;name:string:");
    String columns = "id int,name string";
    assertEquals(" id int", HiveUtility.getColumnsForPartitionedCreateTables(conf, columns));
}

---

From source file:co.nubetech.hiho.hive.TestHiveUtility.java

License:Apache License

@Test(expected = HIHOException.class)
public void testGetCreateQuery() throws HIHOException {
    ColumnInfo intColumn = new ColumnInfo(0, Types.INTEGER, "id");
    ColumnInfo stringColumn = new ColumnInfo(1, Types.VARCHAR, "country");
    ArrayList<ColumnInfo> columns = new ArrayList<ColumnInfo>();
    columns.add(intColumn);/*from   w  w  w . java  2 s  .  c  o m*/
    columns.add(stringColumn); // HiveUtility.// = "employee";
    GenericDBWritable writable = new GenericDBWritable(columns, null);

    // This is normal case to generate basic create query
    Configuration conf = new Configuration();
    conf.set(HIHOConf.HIVE_TABLE_NAME, "employee");
    conf.set(HIHOConf.INPUT_OUTPUT_STRATEGY, "DELIMITED");
    conf.set(HIHOConf.INPUT_OUTPUT_DELIMITER, ";");
    assertEquals(
            "CREATE TABLE `employee` ( `id` int,`country` string) ROW FORMAT DELIMITED FIELDS TERMINATED BY ';' STORED AS TEXTFILE",
            HiveUtility.getCreateQuery(conf, writable));

    // This case show create query when clusteredBy and sortedBy
    // configuration is given user,please make a note sortedBy feature will
    // not work till clusteredBy feature is not defined.
    Configuration conf1 = new Configuration();
    conf1.set(HIHOConf.HIVE_TABLE_NAME, "employee");
    conf1.set(HIHOConf.INPUT_OUTPUT_STRATEGY, "DELIMITED");
    conf1.set(HIHOConf.INPUT_OUTPUT_DELIMITER, ",");
    conf1.set(HIHOConf.INPUT_OUTPUT_DELIMITER, ",");
    conf1.set(HIHOConf.HIVE_CLUSTERED_BY, "name:2");
    conf1.set(HIHOConf.HIVE_SORTED_BY, "name");
    // System.out.println(HiveUtility.getCreateQuery(conf1, writable));
    assertEquals(
            "CREATE TABLE `employee` ( `id` int,`country` string) CLUSTERED BY (name) SORTED BY (name) INTO 2 BUCKETS ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE",
            HiveUtility.getCreateQuery(conf1, writable));

    // This case is when user has defined partitionedBy configuration
    Configuration conf3 = new Configuration();
    conf3.set(HIHOConf.HIVE_TABLE_NAME, "employee");
    conf3.set(HIHOConf.INPUT_OUTPUT_STRATEGY, "DELIMITED");
    conf3.set(HIHOConf.INPUT_OUTPUT_DELIMITER, ",");
    conf3.set(HIHOConf.INPUT_OUTPUT_DELIMITER, ",");
    conf3.set(HIHOConf.HIVE_CLUSTERED_BY, "name:2");
    conf3.set(HIHOConf.HIVE_SORTED_BY, "name");
    conf3.set(HIHOConf.HIVE_PARTITION_BY, "country:string");
    String partitionBy = "country:string";
    // String partitionBy1 = "name:string:raj,country:string";
    // System.out.println(HiveUtility.getCreateQuery(conf3,
    // writable,partitionBy));
    assertEquals(
            "CREATE TABLE `employee` ( `id` int) PARTITIONED BY (country string) CLUSTERED BY (name) SORTED BY (name) INTO 2 BUCKETS ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE",
            HiveUtility.getCreateQuery(conf3, writable, partitionBy));

    // This is case of dynamicPartition when static and dynamic both
    // partitions are defined
    Configuration conf4 = new Configuration();
    conf4.set(HIHOConf.HIVE_TABLE_NAME, "employee");
    conf4.set(HIHOConf.INPUT_OUTPUT_STRATEGY, "DELIMITED");
    conf4.set(HIHOConf.INPUT_OUTPUT_DELIMITER, ",");
    conf4.set(HIHOConf.INPUT_OUTPUT_DELIMITER, ",");
    conf4.set(HIHOConf.HIVE_CLUSTERED_BY, "name:2");
    conf4.set(HIHOConf.HIVE_SORTED_BY, "name");
    conf4.set(HIHOConf.HIVE_PARTITION_BY, "name:string:raj;country:string");
    // String partitionBy = "country:string";
    String partitionBy1 = "name:string:raj;country:string";
    System.out.println(HiveUtility.getCreateQuery(conf4, writable, partitionBy1));
    assertEquals(
            "CREATE TABLE `employee` ( `id` int) PARTITIONED BY (name string,country string) CLUSTERED BY (name) SORTED BY (name) INTO 2 BUCKETS ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE",
            HiveUtility.getCreateQuery(conf4, writable, partitionBy1));

    // This case will throw HIHOException as with clusteredBy conf number of
    // buckets is not defined,it must be defined as name:2
    // please make a note that this case is true for both getCreateQuery()
    // function
    Configuration conf2 = new Configuration();
    conf2.set(HIHOConf.INPUT_OUTPUT_STRATEGY, "DELIMITED");
    conf2.set(HIHOConf.INPUT_OUTPUT_DELIMITER, ",");
    conf2.set(HIHOConf.HIVE_CLUSTERED_BY, "name");
    conf2.set(HIHOConf.HIVE_TABLE_NAME, "employee");
    HiveUtility.getCreateQuery(conf2, writable);

}

---

From source file:co.nubetech.hiho.hive.TestHiveUtility.java

License:Apache License

@Test
public void testGetLoadQuery() throws HIHOException {
    ColumnInfo intColumn = new ColumnInfo(0, Types.INTEGER, "intColumn");
    ColumnInfo stringColumn = new ColumnInfo(1, Types.VARCHAR, "stringColumn");
    ArrayList<ColumnInfo> columns = new ArrayList<ColumnInfo>();
    columns.add(intColumn);/*from ww  w .  j a v  a 2 s.co m*/
    columns.add(stringColumn);
    // HiveUtility.tableName = "employee";
    GenericDBWritable writable = new GenericDBWritable(columns, null);
    Configuration config = new Configuration();
    // String partitionBy = "country:string";
    String partitionBy1 = "country:string:us";
    config.set(HIHOConf.INPUT_OUTPUT_PATH, "/user/nube/tableForHiho");
    config.set(HIHOConf.HIVE_TABLE_NAME, "employee");
    config.set(HIHOConf.HIVE_PARTITION_BY, "country:string:us");
    assertEquals(
            "LOAD DATA INPATH '/user/nube/tableForHiho' OVERWRITE INTO TABLE `employee` PARTITION ( country='us')",
            HiveUtility.getLoadQuery(config, config.get(HIHOConf.INPUT_OUTPUT_PATH), writable, partitionBy1));

    Configuration config1 = new Configuration();
    String partitionBy = "country:string";
    // String partitionBy1 = "country:string:us";
    config1.set(HIHOConf.INPUT_OUTPUT_PATH, "/user/nube/tableForHiho");
    config1.set(HIHOConf.HIVE_TABLE_NAME, "employee");
    // config1.set(HIHOConf.HIVE_PARTITION_BY, "country:string:us");
    assertEquals("LOAD DATA INPATH '/user/nube/tableForHiho' OVERWRITE INTO TABLE `employee`",
            HiveUtility.getLoadQuery(config1, config.get(HIHOConf.INPUT_OUTPUT_PATH), writable));

}

---

From source file:co.nubetech.hiho.job.DBQueryInputJob.java

License:Apache License

public void populateConfiguration(String[] args, Configuration conf) {
    for (int i = 0; i < args.length - 1; i++) {
        if ("-jdbcDriver".equals(args[i])) {
            conf.set(DBConfiguration.DRIVER_CLASS_PROPERTY, args[++i]);
        } else if ("-jdbcUrl".equals(args[i])) {
            conf.set(DBConfiguration.URL_PROPERTY, args[++i]);
        } else if ("-jdbcUsername".equals(args[i])) {
            conf.set(DBConfiguration.USERNAME_PROPERTY, args[++i]);
        } else if ("-jdbcPassword".equals(args[i])) {
            conf.set(DBConfiguration.PASSWORD_PROPERTY, args[++i]);
        } else if ("-inputQuery".equals(args[i])) {
            conf.set(DBConfiguration.INPUT_QUERY, args[++i]);
        } else if ("-inputBoundingQuery".equals(args[i])) {
            conf.set(DBConfiguration.INPUT_BOUNDING_QUERY, args[++i]);
        } else if ("-outputPath".equals(args[i])) {
            conf.set(HIHOConf.INPUT_OUTPUT_PATH, args[++i]);
        } else if ("-outputStrategy".equals(args[i])) {
            conf.set(HIHOConf.INPUT_OUTPUT_STRATEGY, args[++i]);
        } else if ("-delimiter".equals(args[i])) {
            conf.set(HIHOConf.INPUT_OUTPUT_DELIMITER, args[++i]);
        } else if ("-numberOfMappers".equals(args[i])) {
            conf.set(HIHOConf.NUMBER_MAPPERS, args[++i]);
        } else if ("-inputTableName".equals(args[i])) {
            conf.set(DBConfiguration.INPUT_TABLE_NAME_PROPERTY, args[++i]);
        } else if ("-inputFieldNames".equals(args[i])) {
            conf.set(DBConfiguration.INPUT_FIELD_NAMES_PROPERTY, args[++i]);
        } else if ("-inputOrderBy".equals(args[i])) {
            conf.set(DBConfiguration.INPUT_ORDER_BY_PROPERTY, args[++i]);
        } else if ("-inputLoadTo".equals(args[i])) {
            conf.set(HIHOConf.INPUT_OUTPUT_LOADTO, args[++i]);
        } else if ("-inputLoadToPath".equals(args[i])) {
            conf.set(HIHOConf.INPUT_OUTPUT_LOADTO_PATH, args[++i]);
        } else if ("-hiveDriver".equals(args[i])) {
            conf.set(HIHOConf.HIVE_DRIVER, args[++i]);
        } else if ("-hiveUrl".equals(args[i])) {
            conf.set(HIHOConf.HIVE_URL, args[++i]);
        } else if ("-hiveUsername".equals(args[i])) {
            conf.set(HIHOConf.HIVE_USR_NAME, args[++i]);
        } else if ("-hivePassword".equals(args[i])) {
            conf.set(HIHOConf.HIVE_PASSWORD, args[++i]);
        } else if ("-hivePartitionBy".equals(args[i])) {
            conf.set(HIHOConf.HIVE_PARTITION_BY, args[++i]);
        } else if ("-hiveIfNotExists".equals(args[i])) {
            conf.set(HIHOConf.HIVE_TABLE_OVERWRITE, args[++i]);
        } else if ("-hiveTableName".equals(args[i])) {
            conf.set(HIHOConf.HIVE_TABLE_NAME, args[++i]);
        } else if ("-hiveSortedBy".equals(args[i])) {
            conf.set(HIHOConf.HIVE_SORTED_BY, args[++i]);
        } else if ("-hiveClusteredBy".equals(args[i])) {
            conf.set(HIHOConf.HIVE_CLUSTERED_BY, args[++i]);
        }/*www.  j av  a2 s. c o  m*/
    }
}

---

From source file:co.nubetech.hiho.job.DBQueryInputJob.java

License:Apache License

public void checkMandatoryConfs(Configuration conf) throws HIHOException {
    if (conf.get(DBConfiguration.DRIVER_CLASS_PROPERTY) == null) {
        throw new HIHOException("JDBC driver configuration is not specified,please specify JDBC driver class");
    }//w w  w  . j av a  2  s . c om
    if (conf.get(DBConfiguration.URL_PROPERTY) == null) {
        throw new HIHOException("JDBC url path configuration is empty,please specify JDBC url path");
    }
    if (!conf.get(DBConfiguration.DRIVER_CLASS_PROPERTY).contains("hsqldb")) {
        if (conf.get(DBConfiguration.USERNAME_PROPERTY) == null) {
            throw new HIHOException("JDBC user name configuration is empty,please specify JDBC user name");
        }
        if (conf.get(DBConfiguration.PASSWORD_PROPERTY) == null) {
            throw new HIHOException("JDBC password configuration is empty,please specify JDBC password");
        }
    }
    if (conf.get(HIHOConf.INPUT_OUTPUT_PATH) == null) {
        throw new HIHOException("Output path is not specified,please specify output path");
    }
    if (conf.get(HIHOConf.INPUT_OUTPUT_STRATEGY) != null
            && conf.get(HIHOConf.INPUT_OUTPUT_STRATEGY).equals("DELIMITED")) {
        if (conf.get(HIHOConf.INPUT_OUTPUT_DELIMITER) == null) {
            throw new HIHOException("Delimiter is not specified,please specify delimiter");
        }

    }
    if (conf.get(DBConfiguration.INPUT_TABLE_NAME_PROPERTY) == null
            && conf.get(DBConfiguration.INPUT_QUERY) == null) {
        throw new HIHOException(
                "Input table name and input query both configurations are empty, please specify anyone of them");
    }
    if (conf.get(DBConfiguration.INPUT_QUERY) != null
            && conf.get(DBConfiguration.INPUT_BOUNDING_QUERY) == null) {
        throw new HIHOException(
                "Please specify input bounding query as it is mandatory to be defined with input query ");
    }
    if (conf.get(DBConfiguration.INPUT_TABLE_NAME_PROPERTY) != null
            && conf.get(DBConfiguration.INPUT_FIELD_NAMES_PROPERTY) == null) {
        conf.set(DBConfiguration.INPUT_FIELD_NAMES_PROPERTY, "*");
    }

    if (conf.get(HIHOConf.INPUT_OUTPUT_LOADTO) != null && conf.get(HIHOConf.INPUT_OUTPUT_LOADTO_PATH) == null) {
        throw new HIHOException(
                "Load to path configuration is empty, please specify path to load script in loadTOPath configuration");
    }
    if (conf.get(HIHOConf.INPUT_OUTPUT_LOADTO) != null
            && conf.get(HIHOConf.INPUT_OUTPUT_LOADTO).equals("hive")) {
        if (conf.get(HIHOConf.HIVE_URL) == null) {
            throw new HIHOException("The Hive url is not defined, please specify hive url");
        }
        if (conf.get(HIHOConf.HIVE_DRIVER) == null) {
            throw new HIHOException("The Hive driver is not defined, please specify hive driver");
        }
        if (checkForMultiplePartition(conf.get(HIHOConf.HIVE_PARTITION_BY))
                && conf.get(HIHOConf.HIVE_TABLE_NAME) == null) {
            throw new HIHOException("please specify hive table name");
        }

    }

}

---

From source file:co.nubetech.hiho.job.DBQueryInputJob.java

License:Apache License

@Override
public int run(String[] args) throws IOException {

    Configuration conf = getConf();
    populateConfiguration(args, conf);/*from w w  w . j av  a 2 s .com*/

    boolean isMultiplePartition = false;

    if (conf.get(HIHOConf.INPUT_OUTPUT_LOADTO) != null) {
        if (conf.get(HIHOConf.INPUT_OUTPUT_LOADTO).equals("hive")) {
            conf.set("hadoop.job.history.user.location", "none");
            if (conf.get(HIHOConf.HIVE_PARTITION_BY) != null) {
                try {
                    isMultiplePartition = checkForMultiplePartition(conf.get(HIHOConf.HIVE_PARTITION_BY));
                } catch (HIHOException e) {
                    e.printStackTrace();
                }

            }
        }

        if (isMultiplePartition && conf.get(HIHOConf.INPUT_OUTPUT_LOADTO).equals("hive")) {
            populateHiveConfigurationForMultiplePartition(conf);
        } else {
            runJobs(conf, 0);
        }

    } else {
        runJobs(conf, 0);
    }
    return 0;
}

---

From source file:co.nubetech.hiho.job.DBQueryInputJob.java

License:Apache License

public void populateHiveConfigurationForMultiplePartition(Configuration conf) throws IOException {

    String columnName = null, columnType = null;
    ArrayList columnsValues = new ArrayList();

    ArrayList query = new ArrayList();
    ArrayList table = new ArrayList();

    String partitionByData = conf.get(HIHOConf.HIVE_PARTITION_BY);
    boolean isInputQueryDelimited = false;
    String queries = null;//www .j a  v  a  2s. com
    if (conf.get(DBConfiguration.INPUT_QUERY) != null) {
        isInputQueryDelimited = true;
        queries = conf.get(DBConfiguration.INPUT_QUERY);
    }
    // /
    StringTokenizer partitionByTokens = new StringTokenizer(partitionByData, ":");
    columnName = partitionByTokens.nextToken();
    columnType = partitionByTokens.nextToken();
    StringTokenizer partitionByValues = new StringTokenizer(partitionByTokens.nextToken(), ",");
    int counter = 0;
    int tokenCounts = partitionByValues.countTokens();
    while (partitionByValues.hasMoreTokens()) {
        columnsValues.add(counter, partitionByValues.nextToken());
        counter++;
    }
    // /
    if (isInputQueryDelimited) {
        StringTokenizer queryTokens = new StringTokenizer(queries, ";");
        counter = 0;
        while (queryTokens.hasMoreTokens()) {
            query.add(counter, queryTokens.nextToken());
            counter++;
        }
    } else {
        StringTokenizer tableTokens = new StringTokenizer(conf.get(DBConfiguration.INPUT_TABLE_NAME_PROPERTY),
                ";");
        counter = 0;
        while (tableTokens.hasMoreTokens()) {
            table.add(counter, tableTokens.nextToken());
            counter++;
        }

    }
    for (int jobCounter = 0; jobCounter < tokenCounts; jobCounter++) {
        if (isInputQueryDelimited) {
            ;
            conf.set(DBConfiguration.INPUT_QUERY, query.get(jobCounter).toString());
        } else {
            conf.set(DBConfiguration.INPUT_TABLE_NAME_PROPERTY, table.get(jobCounter).toString());
        }
        conf.set(HIHOConf.INPUT_OUTPUT_PATH, conf.get(HIHOConf.INPUT_OUTPUT_PATH) + jobCounter);
        // conf.set(HIHOConf.HIVE_TABLE_OVERWRITE, "true");
        String partitionBy = columnName + ":" + columnType + ":" + columnsValues.get(jobCounter).toString();
        conf.set(HIHOConf.HIVE_PARTITION_BY, partitionBy);
        runJobs(conf, jobCounter);

    }

}

---

From source file:co.nubetech.hiho.job.ExportToDB.java

License:Apache License

public void populateConfiguration(String[] args, Configuration conf) {
    for (int i = 0; i < args.length - 1; i++) {
        if ("-jdbcDriver".equals(args[i])) {
            conf.set(DBConfiguration.DRIVER_CLASS_PROPERTY, args[++i]);
        } else if ("-jdbcUrl".equals(args[i])) {
            conf.set(DBConfiguration.URL_PROPERTY, args[++i]);
        } else if ("-jdbcUsername".equals(args[i])) {
            conf.set(DBConfiguration.USERNAME_PROPERTY, args[++i]);
        } else if ("-jdbcPassword".equals(args[i])) {
            conf.set(DBConfiguration.PASSWORD_PROPERTY, args[++i]);
        } else if ("-delimiter".equals(args[i])) {
            conf.set(HIHOConf.INPUT_OUTPUT_DELIMITER, args[++i]);
        } else if ("-numberOfMappers".equals(args[i])) {
            conf.set(HIHOConf.NUMBER_MAPPERS, args[++i]);
        } else if ("-tableName".equals(args[i])) {
            tableName = args[++i];/*  w  ww. j  av  a  2 s  . com*/
        } else if ("-columnNames".equals(args[i])) {
            columnNames = args[++i];
        } else if ("-inputPath".equals(args[i])) {
            inputPath = args[++i];
        }
    }
}

---

From source file:co.nubetech.hiho.job.ExportToFTPServer.java

License:Apache License

public void populateConfiguration(String[] args, Configuration conf) {
    for (int i = 0; i < args.length - 1; i++) {
        if ("-inputPath".equals(args[i])) {
            inputPath = args[++i];/*  w  w w .j a  v  a 2  s  .c om*/
        } else if ("-outputPath".equals(args[i])) {
            outputPath = args[++i];
        } else if ("-ftpUserName".equals(args[i])) {
            conf.set(HIHOConf.FTP_USER, args[++i]);
        } else if ("-ftpAddress".equals(args[i])) {
            conf.set(HIHOConf.FTP_ADDRESS, args[++i]);
        } else if ("-ftpPortNumper".equals(args[i])) {
            conf.set(HIHOConf.FTP_PORT, args[++i]);
        } else if ("-ftpPassword".equals(args[i])) {
            conf.set(HIHOConf.FTP_PASSWORD, args[++i]);
        }
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »