List of usage examples for org.apache.hadoop.conf Configuration get
public String get(String name, String defaultValue)
name. From source file:co.cask.tephra.distributed.TransactionServiceClient.java
License:Apache License
/** * Create from a configuration. This will first attempt to find a zookeeper * for service discovery. Otherwise it will look for the port in the * config and use localhost./*from w w w . ja va2 s . c om*/ * @param config a configuration containing the zookeeper properties */ @Inject public TransactionServiceClient(Configuration config, ThriftClientProvider clientProvider) { // initialize the retry logic String retryStrat = config.get(TxConstants.Service.CFG_DATA_TX_CLIENT_RETRY_STRATEGY, TxConstants.Service.DEFAULT_DATA_TX_CLIENT_RETRY_STRATEGY); if ("backoff".equals(retryStrat)) { this.retryStrategyProvider = new RetryWithBackoff.Provider(); } else if ("n-times".equals(retryStrat)) { this.retryStrategyProvider = new RetryNTimes.Provider(); } else { String message = "Unknown Retry Strategy '" + retryStrat + "'."; LOG.error(message); throw new IllegalArgumentException(message); } this.retryStrategyProvider.configure(config); LOG.debug("Retry strategy is " + this.retryStrategyProvider); this.clientProvider = clientProvider; }
From source file:co.cask.tephra.inmemory.InMemoryTransactionService.java
License:Apache License
@Inject public InMemoryTransactionService(Configuration conf, DiscoveryService discoveryService, Provider<TransactionManager> txManagerProvider) { this.discoveryService = discoveryService; this.txManagerProvider = txManagerProvider; this.serviceName = conf.get(TxConstants.Service.CFG_DATA_TX_DISCOVERY_SERVICE_NAME, TxConstants.Service.DEFAULT_DATA_TX_DISCOVERY_SERVICE_NAME); address = conf.get(TxConstants.Service.CFG_DATA_TX_BIND_ADDRESS, TxConstants.Service.DEFAULT_DATA_TX_BIND_ADDRESS); port = conf.getInt(TxConstants.Service.CFG_DATA_TX_BIND_PORT, TxConstants.Service.DEFAULT_DATA_TX_BIND_PORT); // Retrieve the number of threads for the service threads = conf.getInt(TxConstants.Service.CFG_DATA_TX_SERVER_THREADS, TxConstants.Service.DEFAULT_DATA_TX_SERVER_THREADS); ioThreads = conf.getInt(TxConstants.Service.CFG_DATA_TX_SERVER_IO_THREADS, TxConstants.Service.DEFAULT_DATA_TX_SERVER_IO_THREADS); maxReadBufferBytes = conf.getInt(TxConstants.Service.CFG_DATA_TX_THRIFT_MAX_READ_BUFFER, TxConstants.Service.DEFAULT_DATA_TX_THRIFT_MAX_READ_BUFFER); LOG.info("Configuring TransactionService" + ", address: " + address + ", port: " + port + ", threads: " + threads + ", io threads: " + ioThreads + ", max read buffer (bytes): " + maxReadBufferBytes); }
From source file:co.nubetech.apache.hadoop.OracleDBRecordReader.java
License:Apache License
/** * Set session time zone/*from w w w.java2 s . c o m*/ * * @param conf * The current configuration. We read the * 'oracle.sessionTimeZone' property from here. * @param conn * The connection to alter the timezone properties of. */ public static void setSessionTimeZone(Configuration conf, Connection conn) throws SQLException { // need to use reflection to call the method setSessionTimeZone on // the OracleConnection class because oracle specific java libraries are // not accessible in this context. Method method; try { method = conn.getClass().getMethod("setSessionTimeZone", new Class[] { String.class }); } catch (Exception ex) { LOG.error("Could not find method setSessionTimeZone in " + conn.getClass().getName(), ex); // rethrow SQLException throw new SQLException(ex); } // Need to set the time zone in order for Java // to correctly access the column "TIMESTAMP WITH LOCAL TIME ZONE". // We can't easily get the correct Oracle-specific timezone string // from Java; just let the user set the timezone in a property. String clientTimeZone = conf.get(SESSION_TIMEZONE_KEY, "GMT"); try { method.setAccessible(true); method.invoke(conn, clientTimeZone); LOG.info("Time zone has been set to " + clientTimeZone); } catch (Exception ex) { LOG.warn("Time zone " + clientTimeZone + " could not be set on Oracle database."); LOG.warn("Setting default time zone: GMT"); try { // "GMT" timezone is guaranteed to exist. method.invoke(conn, "GMT"); } catch (Exception ex2) { LOG.error("Could not set time zone for oracle connection", ex2); // rethrow SQLException throw new SQLException(ex); } } }
From source file:co.nubetech.hiho.hive.HiveUtility.java
License:Apache License
public static String getCreateQuery(Configuration conf, GenericDBWritable writable, String partitionBy) throws HIHOException { StringBuilder builder = new StringBuilder(); String tableName = getTableName(conf); if (conf.get(HIHOConf.HIVE_TABLE_OVERWRITE, "false").equals("true")) { builder.append("CREATE TABLE IF NOT EXISTS `" + tableName + "` ("); } else {//from w ww .j a va2 s . co m builder.append("CREATE TABLE `" + tableName + "` ("); } String columns = getColumns(writable); builder.append(getColumnsForPartitionedCreateTables(conf, columns)); builder.append(") PARTITIONED BY (" + getPartitionBy(partitionBy)); builder.append(")"); builder = appendClusteredByToCreateQuery(conf, builder); builder.append(" ROW FORMAT "); builder = appendDelimitedDataToCreateQuery(conf, builder); return builder.toString(); }
From source file:co.nubetech.hiho.hive.HiveUtility.java
License:Apache License
public static String getCreateQuery(Configuration conf, GenericDBWritable writable) throws HIHOException { StringBuilder builder = new StringBuilder(); String tableName = getTableName(conf); if (conf.get(HIHOConf.HIVE_TABLE_OVERWRITE, "false").equals("true")) { builder.append("CREATE TABLE IF NOT EXISTS `" + tableName + "` ( "); } else {/* w w w . j a v a 2 s . c o m*/ builder.append("CREATE TABLE `" + tableName + "` ( "); } builder.append(getColumns(writable)); builder.append(")"); builder = appendClusteredByToCreateQuery(conf, builder); builder.append(" ROW FORMAT "); builder = appendDelimitedDataToCreateQuery(conf, builder); return builder.toString(); }
From source file:co.nubetech.hiho.mapreduce.lib.output.AppendSequenceFileOutputFormat.java
License:Apache License
@Override public void checkOutputSpecs(JobContext job) throws FileAlreadyExistsException, IOException { try {/*from www .ja va2 s.c o m*/ super.checkOutputSpecs(job); } catch (FileAlreadyExistsException e) { Configuration conf = job.getConfiguration(); isAppend = conf.get(HIHOConf.IS_APPEND, "false"); if (isAppend.equalsIgnoreCase("false")) { throw new FileAlreadyExistsException(); } else { Path outDir = getOutputPath(job); if (outDir == null) { throw new InvalidJobConfException("OUTPUT directory not set."); } } } }
From source file:com.aerospike.hadoop.mapreduce.AerospikeConfigUtil.java
License:Apache License
public static String getInputHost(Configuration conf) { String host = conf.get(INPUT_HOST, DEFAULT_INPUT_HOST); log.info("using " + INPUT_HOST + " = " + host); return host;/*w w w. j a va 2 s . c o m*/ }
From source file:com.aerospike.hadoop.mapreduce.AerospikeConfigUtil.java
License:Apache License
public static String getInputOperation(Configuration conf) { String operation = conf.get(INPUT_OPERATION, DEFAULT_INPUT_OPERATION); if (!operation.equals("scan") && !operation.equals("numrange")) throw new UnsupportedOperationException("input operation must be 'scan' or 'numrange'"); log.info("using " + INPUT_OPERATION + " = " + operation); return operation; }
From source file:com.aerospike.hadoop.mapreduce.AerospikeConfigUtil.java
License:Apache License
public static String getOutputHost(Configuration conf) { String host = conf.get(OUTPUT_HOST, DEFAULT_OUTPUT_HOST); log.info("using " + OUTPUT_HOST + " = " + host); return host;/*from w ww . jav a 2 s . c om*/ }
From source file:com.alexholmes.hadooputils.sort.DelimitedLineRecordReader.java
License:Apache License
protected void initialize(Configuration job, FileSplit split) throws IOException { this.maxLineLength = job.getInt("mapred.linerecordreader.maxlength", Integer.MAX_VALUE); start = split.getStart();/*from w ww .j a v a 2 s .co m*/ end = start + split.getLength(); final Path file = split.getPath(); compressionCodecs = new CompressionCodecFactory(job); final CompressionCodec codec = compressionCodecs.getCodec(file); // open the file and seek to the start of the split FileSystem fs = file.getFileSystem(job); fileIn = fs.open(split.getPath()); boolean skipFirstLine = false; String rowDelim = job.get("textinputformat.record.delimiter", null); if (codec != null) { if (rowDelim != null) { byte[] hexcode = SortConfig.getHexDelimiter(rowDelim); in = new DelimitedLineReader(codec.createInputStream(fileIn), job, (hexcode != null) ? hexcode : rowDelim.getBytes()); } else { in = new DelimitedLineReader(codec.createInputStream(fileIn), job); } end = Long.MAX_VALUE; } else { if (start != 0) { skipFirstLine = true; --start; fileIn.seek(start); } if (rowDelim != null) { byte[] hexcode = SortConfig.getHexDelimiter(rowDelim); in = new DelimitedLineReader(fileIn, job, (hexcode != null) ? hexcode : rowDelim.getBytes()); } else { in = new DelimitedLineReader(fileIn, job); } } if (skipFirstLine) { // skip first line and re-establish "start". start += in.readLine(new Text(), 0, (int) Math.min((long) Integer.MAX_VALUE, end - start)); } this.pos = start; }