List of usage examples for org.apache.hadoop.conf Configuration getLong
public long getLong(String name, long defaultValue)
name property as a long. From source file:com.asakusafw.runtime.stage.input.TemporaryInputFormat.java
License:Apache License
private List<InputSplit> getSplits(Configuration configuration, List<Path> paths) throws IOException { long splitSize = configuration.getLong(KEY_DEFAULT_SPLIT_SIZE, DEFAULT_SPLIT_SIZE); List<InputSplit> results = new ArrayList<>(); for (Path path : paths) { FileSystem fs = path.getFileSystem(configuration); FileStatus[] statuses = fs.globStatus(path); if (statuses == null) { continue; }//from w w w . ja va 2 s . co m for (FileStatus status : statuses) { BlockMap blockMap = BlockMap.create(status.getPath().toString(), status.getLen(), BlockMap.computeBlocks(fs, status), false); results.addAll(computeSplits(status.getPath(), blockMap, splitSize)); } } return results; }
From source file:com.asakusafw.runtime.stage.optimizer.ReducerSimplifierConfigurator.java
License:Apache License
@Override public void configure(Job job) throws IOException, InterruptedException { int count = job.getNumReduceTasks(); if (count <= TASKS_TINY) { return;/* w w w. ja v a 2 s. c o m*/ } Configuration conf = job.getConfiguration(); long limit = conf.getLong(KEY_TINY_LIMIT, -1L); if (limit < 0L) { if (LOG.isDebugEnabled()) { LOG.debug(MessageFormat.format("Reducer simplifier is disabled for tiny inputs: {0}", //$NON-NLS-1$ job.getJobName())); } return; } long estimated = StageInputDriver.estimateInputSize(job); if (LOG.isDebugEnabled()) { LOG.debug(MessageFormat.format("Reducer simplifier: job={0}, tiny-limit={1}, estimated={2}", //$NON-NLS-1$ job.getJobName(), limit, estimated)); } if (estimated < 0L || estimated > limit) { return; } LOG.info(MessageFormat.format("The number of reduce task ({0}) is configured: {1}->{2}", job.getJobName(), job.getNumReduceTasks(), TASKS_TINY)); job.setNumReduceTasks(TASKS_TINY); }
From source file:com.asakusafw.runtime.stage.resource.StageResourceDriver.java
License:Apache License
/** * Adds a resource path into the target job object. * @param job the target job/*from w w w. j a va 2s . com*/ * @param resourcePath the resource path expression (this must be accessible from task execution nodes) * @param resourceName the resource name * @throws IOException if failed to detect resources on the path * @throws IllegalArgumentException if some parameters are {@code null} */ public static void add(Job job, String resourcePath, String resourceName) throws IOException { if (job == null) { throw new IllegalArgumentException("job must not be null"); //$NON-NLS-1$ } if (resourcePath == null) { throw new IllegalArgumentException("resourcePath must not be null"); //$NON-NLS-1$ } if (resourceName == null) { throw new IllegalArgumentException("resourceName must not be null"); //$NON-NLS-1$ } Configuration conf = job.getConfiguration(); List<FileStatus> list = TemporaryStorage.listStatus(conf, new Path(resourcePath)); if (list.isEmpty()) { throw new IOException(MessageFormat.format("Resource not found: {0}", resourcePath)); } List<String> localNames = restoreStrings(conf, getLocalCacheNameKey(resourceName)); List<String> remotePaths = restoreStrings(conf, getRemotePathKey(resourceName)); long size = conf.getLong(KEY_SIZE, 0L); int index = localNames.size(); for (FileStatus status : list) { String name = String.format("%s-%04d", resourceName, index++); //$NON-NLS-1$ StringBuilder buf = new StringBuilder(); buf.append(status.getPath().toString()); buf.append('#'); buf.append(name); String cachePath = buf.toString(); remotePaths.add(status.getPath().toString()); localNames.add(name); try { URI uri = new URI(cachePath); DistributedCache.addCacheFile(uri, conf); } catch (URISyntaxException e) { throw new IllegalStateException(e); } size += status.getLen(); } conf.setStrings(getLocalCacheNameKey(resourceName), localNames.toArray(new String[localNames.size()])); conf.setStrings(getRemotePathKey(resourceName), remotePaths.toArray(new String[remotePaths.size()])); conf.setLong(KEY_SIZE, size); if (JobCompatibility.isLocalMode(job)) { if (LOG.isDebugEnabled()) { LOG.debug("symlinks for distributed cache will not be created in standalone mode"); //$NON-NLS-1$ } } else { DistributedCache.createSymlink(conf); } }
From source file:com.asakusafw.thundergate.runtime.cache.mapreduce.Invalidation.java
License:Apache License
/** * Returns the ending timestamp of each record is invalidate. * @param configuration the current configuration * @return the timestamp - each record is invalidated only if it is older than the timestamp, * may be {@code 0} if invalidation is not enabled *//*from ww w . j a va 2 s .c o m*/ public static long getInvalidationTimestamp(Configuration configuration) { return configuration.getLong(KEY_INTERNAL_TIMESTAMP, 0); }
From source file:com.asp.tranlog.TsvImporterMapper.java
License:Apache License
/** * Handles common parameter initialization that a subclass might want to * leverage.//from w w w . j ava 2s. c om * * @param context */ protected void doSetup(Context context) { Configuration conf = context.getConfiguration(); // If a custom separator has been used, // decode it back from Base64 encoding. separator = conf.get(ImportTsv.SEPARATOR_CONF_KEY); if (separator == null) { separator = ImportTsv.DEFAULT_SEPARATOR; } else { separator = new String(Base64.decode(separator)); } hbase_rowkey_separator = conf.get(ImportTsv.SEPARATOR_CONF_ROWKEY); if (hbase_rowkey_separator == null || hbase_rowkey_separator.trim().length() == 0) { hbase_rowkey_separator = ""; } else { hbase_rowkey_separator = new String(Base64.decode(hbase_rowkey_separator)); } ts = conf.getLong(ImportTsv.TIMESTAMP_CONF_KEY, System.currentTimeMillis()); skipBadLines = context.getConfiguration().getBoolean(ImportTsv.SKIP_LINES_CONF_KEY, true); badLineCount = context.getCounter("ImportTsv", "Bad Lines"); }
From source file:com.bah.culvert.accumulo.database.AccumuloDatabaseAdapter.java
License:Apache License
@Override public TableAdapter getTableAdapter(String tableName) { Configuration conf = this.getConf(); return new AccumuloTableAdapter(this.conn, tableName, conf.getLong(MAX_MEMORY_KEY, DEFAULT_MAX_MEMORY), conf.getLong(MAX_LATENCY_KEY, DEFAULT_MAX_LATENCY), conf.getInt(MAX_THREADS_KEY, DEFAULT_MAX_THREADS)); }
From source file:com.baynote.kafka.hadoop.KafkaInputFormat.java
License:Apache License
/** * Gets the offset timestamp set by {@link #setIncludeOffsetsAfterTimestamp(Job, long)}, returning {@code 0} by * default./* w w w . j av a 2s . com*/ * * @param conf * the job conf. * @return the offset timestamp, {@code 0} by default. */ public static long getIncludeOffsetsAfterTimestamp(final Configuration conf) { return conf.getLong("kafka.timestamp.offset", DEFAULT_INCLUDE_OFFSETS_AFTER_TIMESTAMP); }
From source file:com.buzzinate.dm.cassandra.ColumnFamilyRecordWriter.java
License:Apache License
ColumnFamilyRecordWriter(Configuration conf) throws IOException { this.conf = conf; this.ringCache = new RingCache(conf); this.queueSize = conf.getInt(ColumnFamilyOutputFormat.QUEUE_SIZE, 32 * Runtime.getRuntime().availableProcessors()); this.clients = new HashMap<Range, RangeClient>(); batchThreshold = conf.getLong(ColumnFamilyOutputFormat.BATCH_THRESHOLD, 32); consistencyLevel = ConsistencyLevel.valueOf(ConfigHelper.getWriteConsistencyLevel(conf)); }
From source file:com.ci.backports.hadoop.hbase.ZHFileOutputFormat.java
License:Apache License
public RecordWriter<ImmutableBytesWritable, KeyValue> getRecordWriter(final TaskAttemptContext context) throws IOException, InterruptedException { // Get the path of the temporary output file final Path outputPath = FileOutputFormat.getOutputPath(context); final Path outputdir = new FileOutputCommitter(outputPath, context).getWorkPath(); Configuration conf = context.getConfiguration(); final FileSystem fs = outputdir.getFileSystem(conf); // These configs. are from hbase-*.xml final long maxsize = conf.getLong("hbase.hregion.max.filesize", HConstants.DEFAULT_MAX_FILE_SIZE); final int blocksize = conf.getInt("hfile.min.blocksize.size", HFile.DEFAULT_BLOCKSIZE); // Invented config. Add to hbase-*.xml if other than default compression. final String compression = conf.get("hfile.compression", Compression.Algorithm.NONE.getName()); return new RecordWriter<ImmutableBytesWritable, KeyValue>() { // Map of families to writers and how much has been output on the writer. private final Map<byte[], WriterLength> writers = new TreeMap<byte[], WriterLength>( Bytes.BYTES_COMPARATOR); private byte[] previousRow = HConstants.EMPTY_BYTE_ARRAY; private final byte[] now = Bytes.toBytes(System.currentTimeMillis()); private boolean rollRequested = false; public void write(ImmutableBytesWritable row, KeyValue kv) throws IOException { // null input == user explicitly wants to flush if (row == null && kv == null) { rollWriters();// w w w.j a v a 2 s .com return; } byte[] rowKey = kv.getRow(); long length = kv.getLength(); byte[] family = kv.getFamily(); WriterLength wl = this.writers.get(family); // If this is a new column family, verify that the directory exists if (wl == null) { fs.mkdirs(new Path(outputdir, Bytes.toString(family))); } // If any of the HFiles for the column families has reached // maxsize, we need to roll all the writers if (wl != null && wl.written + length >= maxsize) { this.rollRequested = true; } // This can only happen once a row is finished though if (rollRequested && Bytes.compareTo(this.previousRow, rowKey) != 0) { rollWriters(); } // create a new HLog writer, if necessary if (wl == null || wl.writer == null) { wl = getNewWriter(family); } // we now have the proper HLog writer. full steam ahead kv.updateLatestStamp(this.now); wl.writer.append(kv); wl.written += length; // Copy the row so we know when a row transition. this.previousRow = rowKey; } private void rollWriters() throws IOException { for (WriterLength wl : this.writers.values()) { if (wl.writer != null) { LOG.info("Writer=" + wl.writer.getPath() + ((wl.written == 0) ? "" : ", wrote=" + wl.written)); close(wl.writer); } wl.writer = null; wl.written = 0; } this.rollRequested = false; } /* Create a new HFile.Writer. * @param family * @return A WriterLength, containing a new HFile.Writer. * @throws IOException */ private WriterLength getNewWriter(byte[] family) throws IOException { WriterLength wl = new WriterLength(); Path familydir = new Path(outputdir, Bytes.toString(family)); wl.writer = new HFile.Writer(fs, StoreFile.getUniqueFile(fs, familydir), blocksize, compression, KeyValue.KEY_COMPARATOR); this.writers.put(family, wl); return wl; } private void close(final HFile.Writer w) throws IOException { if (w != null) { w.appendFileInfo(StoreFile.BULKLOAD_TIME_KEY, Bytes.toBytes(System.currentTimeMillis())); w.appendFileInfo(StoreFile.BULKLOAD_TASK_KEY, Bytes.toBytes(context.getTaskAttemptID().toString())); w.appendFileInfo(StoreFile.MAJOR_COMPACTION_KEY, Bytes.toBytes(true)); w.close(); } } public void close(TaskAttemptContext c) throws IOException, InterruptedException { for (WriterLength wl : this.writers.values()) { close(wl.writer); } } }; }
From source file:com.citic.zxyjs.zwlscx.mapreduce.lib.input.HFileOutputFormatBase.java
License:Apache License
public RecordWriter<ImmutableBytesWritable, KeyValue> getRecordWriter(final TaskAttemptContext context) throws IOException, InterruptedException { // Get the path of the temporary output file final Path outputPath = FileOutputFormat.getOutputPath(context); final Path outputdir = new FileOutputCommitter(outputPath, context).getWorkPath(); final Path ignoreOutputPath = new Path(outputPath + "_ignore"); final Configuration conf = context.getConfiguration(); final FileSystem fs = outputdir.getFileSystem(conf); // These configs. are from hbase-*.xml final long maxsize = conf.getLong(HConstants.HREGION_MAX_FILESIZE, HConstants.DEFAULT_MAX_FILE_SIZE); // Invented config. Add to hbase-*.xml if other than default // compression. final String defaultCompression = conf.get("hfile.compression", Compression.Algorithm.NONE.getName()); final boolean compactionExclude = conf.getBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude", false);/*from w w w . jav a 2s. c o m*/ if (fs.exists(ignoreOutputPath)) { LOG.info("Deleted " + ignoreOutputPath.toString() + " success."); fs.delete(ignoreOutputPath, true); } // create a map from column family to the compression algorithm final Map<byte[], String> compressionMap = createFamilyCompressionMap(conf); final Map<byte[], String> bloomTypeMap = createFamilyBloomMap(conf); final Map<byte[], String> blockSizeMap = createFamilyBlockSizeMap(conf); String dataBlockEncodingStr = conf.get(DATABLOCK_ENCODING_CONF_KEY); final HFileDataBlockEncoder encoder; if (dataBlockEncodingStr == null) { encoder = NoOpDataBlockEncoder.INSTANCE; } else { try { encoder = new HFileDataBlockEncoderImpl(DataBlockEncoding.valueOf(dataBlockEncodingStr)); } catch (IllegalArgumentException ex) { throw new RuntimeException("Invalid data block encoding type configured for the param " + DATABLOCK_ENCODING_CONF_KEY + " : " + dataBlockEncodingStr); } } return new RecordWriter<ImmutableBytesWritable, KeyValue>() { // Map of families to writers and how much has been output on the // writer. private final Map<byte[], WriterLength> writers = new TreeMap<byte[], WriterLength>( Bytes.BYTES_COMPARATOR); private final FSDataOutputStream dos = fs.create(ignoreOutputPath); private byte[] previousRow = HConstants.EMPTY_BYTE_ARRAY; private final byte[] now = Bytes.toBytes(System.currentTimeMillis()); private boolean rollRequested = false; public void write(ImmutableBytesWritable row, KeyValue kv) throws IOException { // null input == user explicitly wants to flush if (row == null && kv == null) { rollWriters(); return; } byte[] rowKey = kv.getRow(); long length = kv.getLength(); byte[] family = kv.getFamily(); if (ignore(kv)) { byte[] readBuf = rowKey; dos.write(readBuf, 0, readBuf.length); dos.write(Bytes.toBytes("\n")); return; } WriterLength wl = this.writers.get(family); // If this is a new column family, verify that the directory // exists if (wl == null) { Path path = null; path = new Path(outputdir, Bytes.toString(family)); fs.mkdirs(path); } // If any of the HFiles for the column families has reached // maxsize, we need to roll all the writers if (wl != null && wl.written + length >= maxsize) { this.rollRequested = true; } // This can only happen once a row is finished though if (rollRequested && Bytes.compareTo(this.previousRow, rowKey) != 0) { rollWriters(); } // create a new HLog writer, if necessary if (wl == null || wl.writer == null) { wl = getNewWriter(family, conf); } // we now have the proper HLog writer. full steam ahead kv.updateLatestStamp(this.now); wl.writer.append(kv); wl.written += length; // Copy the row so we know when a row transition. this.previousRow = rowKey; } private void rollWriters() throws IOException { for (WriterLength wl : this.writers.values()) { if (wl.writer != null) { LOG.info("Writer=" + wl.writer.getPath() + ((wl.written == 0) ? "" : ", wrote=" + wl.written)); close(wl.writer); } wl.writer = null; wl.written = 0; } this.rollRequested = false; } /* * Create a new StoreFile.Writer. * @param family * @return A WriterLength, containing a new StoreFile.Writer. * @throws IOException */ private WriterLength getNewWriter(byte[] family, Configuration conf) throws IOException { WriterLength wl = new WriterLength(); Path familydir = new Path(outputdir, Bytes.toString(family)); String compression = compressionMap.get(family); compression = compression == null ? defaultCompression : compression; String bloomTypeStr = bloomTypeMap.get(family); BloomType bloomType = BloomType.NONE; if (bloomTypeStr != null) { bloomType = BloomType.valueOf(bloomTypeStr); } String blockSizeString = blockSizeMap.get(family); int blockSize = blockSizeString == null ? HConstants.DEFAULT_BLOCKSIZE : Integer.parseInt(blockSizeString); Configuration tempConf = new Configuration(conf); tempConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f); wl.writer = new StoreFile.WriterBuilder(conf, new CacheConfig(tempConf), fs, blockSize) .withOutputDir(familydir) .withCompression(AbstractHFileWriter.compressionByName(compression)) .withBloomType(bloomType).withComparator(KeyValue.COMPARATOR).withDataBlockEncoder(encoder) .withChecksumType(HStore.getChecksumType(conf)) .withBytesPerChecksum(HStore.getBytesPerChecksum(conf)).build(); this.writers.put(family, wl); return wl; } private void close(final StoreFile.Writer w) throws IOException { if (w != null) { w.appendFileInfo(StoreFile.BULKLOAD_TIME_KEY, Bytes.toBytes(System.currentTimeMillis())); w.appendFileInfo(StoreFile.BULKLOAD_TASK_KEY, Bytes.toBytes(context.getTaskAttemptID().toString())); w.appendFileInfo(StoreFile.MAJOR_COMPACTION_KEY, Bytes.toBytes(true)); w.appendFileInfo(StoreFile.EXCLUDE_FROM_MINOR_COMPACTION_KEY, Bytes.toBytes(compactionExclude)); w.appendTrackedTimestampsToMetadata(); w.close(); } } public void close(TaskAttemptContext c) throws IOException, InterruptedException { dos.flush(); dos.close(); for (WriterLength wl : this.writers.values()) { close(wl.writer); } } }; }