List of usage examples for org.apache.hadoop.conf Configuration getInt
public int getInt(String name, int defaultValue)
name property as an int. From source file:com.baynote.kafka.hadoop.KafkaInputFormat.java
License:Apache License
/** * Gets the Kafka fetch size set by {@link #setKafkaFetchSizeBytes(Job, int)}, defaulting to * {@link #DEFAULT_FETCH_SIZE_BYTES} if it has not been set. * /*from ww w . java2 s . c o m*/ * @param conf * the job conf. * @return the Kafka fetch size. */ public static int getKafkaFetchSizeBytes(final Configuration conf) { return conf.getInt("kafka.fetch.size", DEFAULT_FETCH_SIZE_BYTES); }
From source file:com.baynote.kafka.hadoop.KafkaInputFormat.java
License:Apache License
/** * Gets the Kafka buffer size set by {@link #setKafkaBufferSizeBytes(Job, int)}, defaulting to * {@link #DEFAULT_BUFFER_SIZE_BYTES} if it has not been set. * /* w w w . j av a 2s . c om*/ * @param conf * the job conf. * @return the Kafka buffer size. */ public static int getKafkaBufferSizeBytes(final Configuration conf) { return conf.getInt("kafka.socket.buffersize", DEFAULT_BUFFER_SIZE_BYTES); }
From source file:com.baynote.kafka.hadoop.KafkaInputFormat.java
License:Apache License
/** * Gets the Kafka socket timeout set by {@link #setKafkaSocketTimeoutMs(Job, int)}, defaulting to * {@link #DEFAULT_SOCKET_TIMEOUT_MS} if it has not been set. * //from w ww. j a va2s . c o m * @param conf * the job conf. * @return the Kafka socket timeout. */ public static int getKafkaSocketTimeoutMs(final Configuration conf) { return conf.getInt("kafka.socket.timeout.ms", DEFAULT_SOCKET_TIMEOUT_MS); }
From source file:com.bigstep.datalake.DLFileSystem.java
License:Apache License
@Override public synchronized void initialize(URI uri, Configuration conf) throws IOException { super.initialize(uri, conf); uri = selectDatalakeEndpointURI(uri, conf); /* set user pattern based on configuration file */ UserParam.setUserPattern(conf.get(DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY, DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT)); kerberosIdentity = initialiseKerberosIdentity(conf); this.shouldUseEncryption = conf.getBoolean(FS_DL_IMPL_SHOULD_USE_ENCRYPTION_CONFIG_NAME, false); if (this.shouldUseEncryption) { initialiseAesEncryption(conf);//from ww w .java2 s . c om } this.homeDirectory = conf.get(FS_DL_IMPL_HOME_DIRECTORY); if (homeDirectory == null) throw new IOException( "The Datalake requires a home directory to be configured in the fs.dl.impl.homeDirectory configuration variable. This is in the form /data_lake/dlxxxx"); this.defaultEndpoint = conf.get(FS_DL_IMPL_DEFAULT_ENDPOINT); if (defaultEndpoint == null) throw new IOException( "The Datalake requires a default endpoint to be configured the fs.dl.impl.defaultEndpoint configuration variable. This is in the form /data_lake/dlxxxx"); URI defaultEndpointURI = URI.create(defaultEndpoint); String authority = uri.getAuthority() == null ? defaultEndpointURI.getAuthority() : uri.getAuthority(); this.baseUri = URI.create(uri.getScheme() + "://" + authority + this.homeDirectory); this.nnAddrs = resolveNNAddr(); LOG.debug("Created kerberosIdentity " + kerberosIdentity + " for " + this.baseUri); boolean isHA = HAUtil.isClientFailoverConfigured(conf, this.baseUri); boolean isLogicalUri = isHA && HAUtil.isLogicalUri(conf, this.baseUri); // In non-HA or non-logical URI case, the code needs to call // getCanonicalUri() in order to handle the case where no port is // specified in the URI this.tokenServiceName = isLogicalUri ? HAUtil.buildTokenServiceForLogicalUri(this.baseUri, getScheme()) : SecurityUtil.buildTokenService(getCanonicalUri()); if (!isHA) { this.retryPolicy = RetryUtils.getDefaultRetryPolicy(conf, DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_ENABLED_KEY, DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_ENABLED_DEFAULT, DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_SPEC_KEY, DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_SPEC_DEFAULT, SafeModeException.class); } else { int maxFailoverAttempts = conf.getInt(DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY, DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT); int maxRetryAttempts = conf.getInt(DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_MAX_ATTEMPTS_KEY, DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT); int failoverSleepBaseMillis = conf.getInt(DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY, DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT); int failoverSleepMaxMillis = conf.getInt(DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY, DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT); this.retryPolicy = RetryPolicies.failoverOnNetworkException(RetryPolicies.TRY_ONCE_THEN_FAIL, maxFailoverAttempts, maxRetryAttempts, failoverSleepBaseMillis, failoverSleepMaxMillis); } this.workingDir = getHomeDirectory(); //Delegation tokens don't work with httpfs this.canRefreshDelegationToken = false; this.disallowFallbackToInsecureCluster = !conf.getBoolean( CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY, CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_DEFAULT); this.delegationToken = null; this.defaultFilePermissions = Short .decode(conf.get(FS_DL_IMPL_DEFAULT_FILE_PERMISSIONS, this.DEFAULT_FILE_PERMISSIONS)); this.defaultUMask = Short.decode(conf.get(FS_DL_IMPL_DEFAULT_UMASK, this.DEFAULT_UMASK)); this.transportScheme = conf.get(FS_DL_IMPL_TRANSPORT_SCHEME_CONFIG_NAME, FS_DL_IMPL_DEFAULT_TRANSPORT_SCHEME); if (!checkJCE()) throw new IOException(JCE_ERROR); }
From source file:com.bonc.mr_roamRecognition_hjpt.comm.PathRecordReader.java
License:Apache License
public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException { FileSplit split = (FileSplit) genericSplit; Configuration job = context.getConfiguration(); this.maxLineLength = job.getInt(MAX_LINE_LENGTH, Integer.MAX_VALUE); start = split.getStart();/*from ww w . ja v a2 s.c o m*/ end = start + split.getLength(); final Path file = split.getPath(); path = split.getPath().toString(); // open the file and seek to the start of the split final FileSystem fs = file.getFileSystem(job); fileIn = fs.open(file); CompressionCodec codec = new CompressionCodecFactory(job).getCodec(file); if (null != codec) { isCompressedInput = true; decompressor = CodecPool.getDecompressor(codec); if (codec instanceof SplittableCompressionCodec) { final SplitCompressionInputStream cIn = ((SplittableCompressionCodec) codec).createInputStream( fileIn, decompressor, start, end, SplittableCompressionCodec.READ_MODE.BYBLOCK); in = new CompressedSplitLineReader(cIn, job, this.recordDelimiterBytes); start = cIn.getAdjustedStart(); end = cIn.getAdjustedEnd(); filePosition = cIn; } else { in = new SplitLineReader(codec.createInputStream(fileIn, decompressor), job, this.recordDelimiterBytes); filePosition = fileIn; } } else { fileIn.seek(start); in = new SplitLineReader(fileIn, job, this.recordDelimiterBytes); filePosition = fileIn; } // If this is not the first split, we always throw away first record // because we always (except the last split) read one extra line in // next() method. if (start != 0) { start += in.readLine(new Text(), 0, maxBytesToConsume(start)); } this.pos = start; }
From source file:com.buzzinate.dm.cassandra.ColumnFamilyRecordWriter.java
License:Apache License
ColumnFamilyRecordWriter(Configuration conf) throws IOException { this.conf = conf; this.ringCache = new RingCache(conf); this.queueSize = conf.getInt(ColumnFamilyOutputFormat.QUEUE_SIZE, 32 * Runtime.getRuntime().availableProcessors()); this.clients = new HashMap<Range, RangeClient>(); batchThreshold = conf.getLong(ColumnFamilyOutputFormat.BATCH_THRESHOLD, 32); consistencyLevel = ConsistencyLevel.valueOf(ConfigHelper.getWriteConsistencyLevel(conf)); }
From source file:com.chinamobile.bcbsp.bspcontroller.JobInProgress.java
License:Apache License
/** * jobInProgress construct method.//from w ww . java 2 s . c o m * @param jobId * BSP job id. * @param jobFile * BSP job file. * @param controller * BSP controller * @param conf * BSP System configuration. * @throws IOException * exceptions during handle BSP job file. */ public JobInProgress(BSPJobID jobId, Path jobFile, BSPController controller, Configuration conf) throws IOException { this.jobId = jobId; // this.localFs = FileSystem.getLocal(conf); this.BSPlocalFs = new BSPLocalFileSystemImpl(conf); this.jobFile = jobFile; this.controller = controller; // this.status = new JobStatus(jobId, null, 0L, 0L, // JobStatus.State.PREP.value()); this.startTime = System.currentTimeMillis(); //ljn SGA_Graph this.superStepCounter = -2; // this.superStepCounter = 0; this.maxAttemptRecoveryCounter = conf.getInt(Constants.BC_BSP_JOB_RECOVERY_ATTEMPT_MAX, 0); this.maxStaffAttemptRecoveryCounter = conf.getInt(Constants.BC_BSP_STAFF_RECOVERY_ATTEMPT_MAX, 0); this.localJobFile = controller .getLocalPath(Constants.BC_BSP_LOCAL_SUBDIR_CONTROLLER + "/" + jobId + ".xml"); this.localJarFile = controller .getLocalPath(Constants.BC_BSP_LOCAL_SUBDIR_CONTROLLER + "/" + jobId + ".jar"); Path jobDir = controller.getSystemDirectoryForJob(jobId); // FileSystem fs = jobDir.getFileSystem(conf); BSPFileSystem bspfs = new BSPFileSystemImpl(controller, jobId, conf); bspfs.copyToLocalFile(jobFile, localJobFile); job = new BSPJob(jobId, localJobFile.toString()); this.conf = job.getConf(); this.numBSPStaffs = job.getNumBspStaff(); this.profile = new JobProfile(job.getUser(), jobId, jobFile.toString(), job.getJobName()); String jarFile = job.getJar(); if (jarFile != null) { bspfs.copyToLocalFile(new BSPHdfsImpl().newPath(jarFile), localJarFile); } // chang by cui this.priority = job.getPriority(); this.superStepNum = job.getNumSuperStep(); this.status = new JobStatus(jobId, null, 0L, 0L, 0L, JobStatus.State.PREP.value(), this.superStepNum); // new JobStatus(jobId, null, 0L, 0L, JobStatus.State.PREP.value()); this.status.setUsername(job.getUser()); this.status.setStartTime(startTime); // added by feng this.edgeNum = job.getOutgoingEdgeNum(); setCheckPointFrequency(); // For aggregation. /** Add the user program jar to the system's classpath. */ ClassLoaderUtil.addClassPath(localJarFile.toString()); loadAggregators(); this.gssc = new GeneralSSController(jobId); }
From source file:com.chinamobile.bcbsp.workermanager.WorkerAgentForJob.java
License:Apache License
/** * Constructor./*from w w w. j a v a2 s . c o m*/ * * @param conf * Configuration * @param jobId * BSPJobID * @param jobConf * BSPJob * @param workerManager * WorkerManager */ public WorkerAgentForJob(Configuration conf, BSPJobID jobId, BSPJob jobConf, WorkerManager workerManager) throws IOException { this.jobId = jobId; this.jobConf = jobConf; this.workerManager = workerManager; this.workerManagerName = conf.get(Constants.BC_BSP_WORKERAGENT_HOST, Constants.BC_BSP_WORKERAGENT_HOST); this.wssc = new WorkerSSController(jobId, this.workerManagerName); this.conf = conf; String bindAddress = conf.get(Constants.BC_BSP_WORKERAGENT_HOST, Constants.DEFAULT_BC_BSP_WORKERAGENT_HOST); int bindPort = conf.getInt(Constants.BC_BSP_WORKERAGENT_PORT, Constants.DEFAULT_BC_BSP_WORKERAGENT_PORT); bindPort = bindPort + Integer.parseInt(jobId.toString().substring(17)); portForJob = bindPort; workAddress = new InetSocketAddress(bindAddress, bindPort); reinitialize(); // For Aggregation loadAggregators(); }
From source file:com.chinnu.churndetection.fuzzykmeans.FuzzyKMeansMapper.java
@Override protected void setup(Mapper<LongWritable, Text, IntWritable, Vector>.Context context) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); CENTERS = conf.get(Constants.CENTER_TEXT); STARTINDEX = conf.getInt(Constants.STARTINDEX, 0); ENDINDEX = conf.getInt(Constants.ENDINDEX, 0); CLASSINDEX = conf.getInt(Constants.CLASSINDEX, 0); DATALENGTH = ENDINDEX - STARTINDEX;/* w ww. java 2 s . c om*/ super.setup(context); }
From source file:com.chinnu.churndetection.fuzzykmeans.FuzzyKMeansReducer.java
@Override protected void setup(Reducer<IntWritable, Vector, IntWritable, Text>.Context context) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); NEW_CENTER = conf.get(Constants.NEXTCENTER); CURR_CENTER = conf.get(Constants.CENTER_TEXT); STARTINDEX = conf.getInt(Constants.STARTINDEX, 0); ENDINDEX = conf.getInt(Constants.ENDINDEX, 0); DATALENGTH = ENDINDEX - STARTINDEX;//ww w. ja v a 2 s.c o m super.setup(context); }