Example usage for org.apache.hadoop.conf Configuration get

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration get.

Prototype

public String get(String name) 

Source Link

Document

Get the value of the name property, null if no such property exists.

Usage

From source file:co.nubetech.crux.pool.TestHBaseConnectionPoolFactory.java

License:Apache License

@Test
public void testGetConfiguration() {
    Connection connection = new Connection();

    ConnectionProperty property = new ConnectionProperty();
    property.setProperty(CruxConstants.HBASE_ZOOKEEPER_PROPERTY);
    property.setValue("h1:2181");
    connection.addProperty(property);//from   ww w. ja  v  a  2 s. c om

    HBaseConnectionPoolFactory hbaseConnectionPoolFactory = new HBaseConnectionPoolFactory();
    Utility utility = new Utility();
    Configuration conf = utility.getConfiguration(connection);
    assertEquals(conf.get("hbase.zookeeper.quorum"), "h1");
    assertEquals(conf.get("hbase.zookeeper.property.clientPort"), "2181");
}

---

From source file:co.nubetech.hiho.dedup.DelimitedLineRecordReader.java

License:Apache License

/**
 * //from w  ww. j ava2 s .  c  o m
 * @param delimiter
 * @param column
 * 
 * 
 */

@Override
public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException {
    FileSplit split = (FileSplit) genericSplit;
    Configuration job = context.getConfiguration();
    this.delimiter = job.get(DelimitedTextInputFormat.DELIMITER_CONF);
    this.column = job.getInt(DelimitedTextInputFormat.COLUMN_CONF, 0);
    this.maxLineLength = job.getInt("mapred.linerecordreader.maxlength", Integer.MAX_VALUE);
    start = split.getStart();
    end = start + split.getLength();
    final Path file = split.getPath();
    compressionCodecs = new CompressionCodecFactory(job);
    final CompressionCodec codec = compressionCodecs.getCodec(file);

    // open the file and seek to the start of the split
    FileSystem fs = file.getFileSystem(job);
    FSDataInputStream fileIn = fs.open(split.getPath());
    boolean skipFirstLine = false;
    if (codec != null) {
        in = new LineReader(codec.createInputStream(fileIn), job);
        end = Long.MAX_VALUE;
    } else {
        if (start != 0) {
            skipFirstLine = true;
            --start;
            fileIn.seek(start);
        }
        in = new LineReader(fileIn, job);
    }
    if (skipFirstLine) { // skip first line and re-establish "start".
        start += in.readLine(new Text(), 0, (int) Math.min((long) Integer.MAX_VALUE, end - start));
    }
    this.pos = start;
}

---

From source file:co.nubetech.hiho.hive.HiveUtility.java

License:Apache License

public static void createTable(Configuration conf, Job job, GenericDBWritable writable, int jobCounter)
        throws HIHOException {
    String createQuery = null;/*from  w w  w  .j a  va  2s .c  o m*/
    String loadQuery = null;
    String tableName = getTableName(conf);
    FileOutputStream fos = null;
    boolean isCreateTable = true;
    boolean isDyanamicPartition = false;
    if (jobCounter > 0) {
        isCreateTable = false;
    }

    try {

        fos = getFileOutputStream(conf, jobCounter, tableName);
        BufferedWriter w = new BufferedWriter(new OutputStreamWriter(fos));

        String partitionBy = conf.get(HIHOConf.HIVE_PARTITION_BY);
        // tableName = getTableName(conf);

        if (conf.get(HIHOConf.HIVE_PARTITION_BY) == null) {

            if (isCreateTable) {
                createQuery = getCreateQuery(conf, writable);
                w.append(createQuery + "\n");
            }
            loadQuery = getLoadQuery(conf, conf.get(HIHOConf.INPUT_OUTPUT_PATH), writable);
            w.append(loadQuery + "\n");
            logger.warn("\nThe queries are: " + createQuery + "\n" + loadQuery);
        } else {
            loadQuery = getLoadQuery(conf, conf.get(HIHOConf.INPUT_OUTPUT_PATH), writable, partitionBy);
            if (isCreateTable) {
                createQuery = getCreateQuery(conf, writable, partitionBy);
                w.append(createQuery + "\n");
            }
            w.append(loadQuery + "\n");
            logger.warn("\nThe queries are: " + createQuery + "\n" + loadQuery);
            if (getDynamicPartitionBy(partitionBy) != null) {
                isDyanamicPartition = true;
            }
        }

        if (!isDyanamicPartition) {
            runQuery(createQuery, loadQuery, isCreateTable, conf);
        } else {

            String insertQuery = getInsertQueryFromTmpToMain(conf, writable, partitionBy);
            String tmpCreateQuery = getTmpCreateQuery(conf, writable);
            runQuery(tmpCreateQuery, insertQuery, createQuery, loadQuery, conf, isCreateTable);
            w.append(tmpCreateQuery + "\n" + loadQuery + "\n");
            if (isCreateTable) {
                w.append(createQuery + "\n");
            }
            w.append(insertQuery + "\n");
            logger.warn("\nThe queries are: " + createQuery + "\n" + loadQuery + "\n" + tmpCreateQuery + "\n"
                    + insertQuery);
        }
        // w.append(createQuery + "\n" + loadQuery);
        w.close();
        fos.close();
    } catch (Exception e) {
        e.printStackTrace();
        throw new HIHOException("Could not generate hive table", e);
    }
}

---

From source file:co.nubetech.hiho.hive.HiveUtility.java

License:Apache License

public static FileOutputStream getFileOutputStream(Configuration conf, int jobCounter, String tableName)
        throws HIHOException {
    FileOutputStream fos = null;//from   w  w w.j  a v a2  s  . c  o  m
    try {
        if (jobCounter == 0) {
            File file = new File(new File(conf.get(HIHOConf.INPUT_OUTPUT_LOADTO_PATH)),
                    "hiveScript" + tableName + ".txt");

            fos = new FileOutputStream(file);

        } else {
            String path = conf.get(HIHOConf.INPUT_OUTPUT_LOADTO_PATH) + "hiveScript" + tableName + ".txt";
            fos = new FileOutputStream(path, true);
        }
    } catch (FileNotFoundException e) {
        throw new HIHOException();
    }
    return fos;
}

---

From source file:co.nubetech.hiho.hive.HiveUtility.java

License:Apache License

public static String getTableName(Configuration conf) throws HIHOException {
    String tableName = conf.get(HIHOConf.HIVE_TABLE_NAME);
    // if user did not specify hive table name, lets try to deduce it
    // automatically
    // get it from the name of the table we are querying against
    if (tableName == null) {
        tableName = conf.get(DBConfiguration.INPUT_TABLE_NAME_PROPERTY);
    }/*  ww  w. j  a  v  a 2 s.co m*/
    if (tableName == null) {
        if (conf.get(DBConfiguration.INPUT_QUERY) != null) {
            String query = conf.get(DBConfiguration.INPUT_QUERY);
            StringTokenizer queryTokens = new StringTokenizer(query, " ");
            while (queryTokens.hasMoreTokens()) {
                if (queryTokens.nextToken().equalsIgnoreCase("FROM")) {
                    tableName = queryTokens.nextToken();
                }
            }
        }
    }

    // we are missing a couple of cases here. FROM table1, table2. FROM
    // table1 inner join blah blah
    // lets leave them for now and address them when the user need is
    // clearer

    /*
     * if (conf.get(DBConfiguration.INPUT_TABLE_NAME_PROPERTY) != null) {
     * tableName = conf.get(DBConfiguration.INPUT_TABLE_NAME_PROPERTY); }
     * else if (conf.get(DBConfiguration.INPUT_QUERY) != null) { String
     * query = conf.get(DBConfiguration.INPUT_QUERY); StringTokenizer
     * queryTokens = new StringTokenizer(query, " "); while
     * (queryTokens.hasMoreTokens()) { if
     * (queryTokens.nextToken().equalsIgnoreCase("FROM")) { tableName =
     * queryTokens.nextToken(); } } } if
     * (conf.get(HIHOConf.HIVE_MULTIPLE_PARTITION_BY) != null &&
     * conf.get(HIHOConf.HIVE_TABLE_NAME) != null) { tableName =
     * conf.get(HIHOConf.HIVE_TABLE_NAME); }
     */
    if (tableName == null) {
        throw new HIHOException("Cannot get hive table name");
    }
    return tableName;
}

---

From source file:co.nubetech.hiho.hive.HiveUtility.java

License:Apache License

public static String getTmpCreateQuery(Configuration conf, GenericDBWritable writable) throws HIHOException {
    StringBuilder builder = new StringBuilder();
    builder.append("CREATE TABLE `" + getTableName(conf) + "tmp` ( ");

    builder.append(getColumns(writable));

    builder.append(") ROW FORMAT ");
    OutputStrategyEnum outputStrategy = OutputStrategyEnum.valueOf(conf.get(HIHOConf.INPUT_OUTPUT_STRATEGY));
    if (outputStrategy == OutputStrategyEnum.DELIMITED) {
        builder.append("DELIMITED FIELDS TERMINATED BY '");
        builder.append(conf.get(HIHOConf.INPUT_OUTPUT_DELIMITER));
        builder.append("' STORED AS TEXTFILE");
    }/*from w ww  .jav a  2 s . com*/

    return builder.toString();
}

---

From source file:co.nubetech.hiho.hive.HiveUtility.java

License:Apache License

public static String getColumnsForPartitionedCreateTables(Configuration conf, String columns) {
    // columns format columnsName columnType,columnsName1 columnType1
    // get the column by which we want to do partition by
    // and remove that from the table creation
    String dynamicPartitionBy = getDynamicPartitionBy(conf.get(HIHOConf.HIVE_PARTITION_BY));
    StringTokenizer columnTokens = new StringTokenizer(columns, ",");
    columns = " ";
    int count = 0;
    while (columnTokens.hasMoreTokens()) {
        String columnData = columnTokens.nextToken();
        StringTokenizer columnDataTokens = new StringTokenizer(columnData, "` ");
        String columnName = columnDataTokens.nextToken();
        if (!columnName.equals(dynamicPartitionBy) && count > 0) {
            columns = columns + ",";
        }//from w  w w.  j ava 2 s. c om
        if (!columnName.equals(dynamicPartitionBy)) {
            columns = columns + columnData;
            count++;
        }

    }
    return columns;
}

---

From source file:co.nubetech.hiho.hive.HiveUtility.java

License:Apache License

public static StringBuilder appendDelimitedDataToCreateQuery(Configuration conf, StringBuilder builder) {
    OutputStrategyEnum outputStrategy = OutputStrategyEnum.valueOf(conf.get(HIHOConf.INPUT_OUTPUT_STRATEGY));
    if (outputStrategy == OutputStrategyEnum.DELIMITED) {
        builder.append("DELIMITED FIELDS TERMINATED BY '");
        builder.append(conf.get(HIHOConf.INPUT_OUTPUT_DELIMITER));
        builder.append("' STORED AS TEXTFILE");
    }// w ww. jav  a2s. c  om
    return builder;
}

---

From source file:co.nubetech.hiho.hive.HiveUtility.java

License:Apache License

public static StringBuilder appendClusteredByToCreateQuery(Configuration conf, StringBuilder builder)
        throws HIHOException {
    if (conf.get(HIHOConf.HIVE_CLUSTERED_BY) != null) {
        StringTokenizer clusterData = new StringTokenizer(conf.get(HIHOConf.HIVE_CLUSTERED_BY), ":");
        builder.append(" CLUSTERED BY (" + clusterData.nextToken());
        builder.append(")");
        if (conf.get(HIHOConf.HIVE_SORTED_BY) != null) {
            builder.append(" SORTED BY (" + conf.get(HIHOConf.HIVE_SORTED_BY));
            builder.append(")");
        }/*from ww  w.j av  a2  s  . c  o  m*/
        try {
            String buckets = clusterData.nextToken();
            if (buckets == null) {
                throw new HIHOException("The number of buckets wih clustered by is not defined");

            }
            builder.append(" INTO " + buckets + " BUCKETS");
        } catch (NoSuchElementException e) {
            throw new HIHOException("The number of buckets wih clustered by is not defined");

        }

    }

    return builder;

}

---

From source file:co.nubetech.hiho.hive.HiveUtility.java

License:Apache License

public static String getLoadQuery(Configuration conf, String hdfsDir, GenericDBWritable writable,
        String partitionBy) throws HIHOException {
    String loadQuery = getLoadQuery(conf, hdfsDir, writable);
    StringTokenizer tempToken = new StringTokenizer(partitionBy, ";");

    String columnsData = " ";
    while (tempToken.hasMoreTokens()) {
        StringTokenizer columnTokens = new StringTokenizer(tempToken.nextToken(), ":");
        String columnNames = columnTokens.nextToken();
        columnsData = columnsData + columnNames;
        columnTokens.nextToken();/*from  w w  w  .  j  av  a 2s . c  om*/
        if (columnTokens.hasMoreTokens()) {
            columnsData = columnsData + "='" + columnTokens.nextToken() + "'";
        }
        if (tempToken.hasMoreTokens()) {
            columnsData = columnsData + ",";
        }
    }
    if (getDynamicPartitionBy(conf.get(HIHOConf.HIVE_PARTITION_BY)) == null) {
        loadQuery = loadQuery + "` PARTITION (" + columnsData + ")";
    } else {
        loadQuery = loadQuery + "tmp`";
    }

    return loadQuery;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• »