List of usage examples for org.apache.hadoop.conf Configuration getLong
public long getLong(String name, long defaultValue)
name property as a long. From source file:mvm.rya.indexing.accumulo.ConfigUtils.java
License:Apache License
public static long getWriterMaxMemory(Configuration conf) { return conf.getLong(CLOUDBASE_WRITER_MAX_MEMORY, WRITER_MAX_MEMORY); }
From source file:mzb.Balancer.java
License:Apache License
/** * Balance all namenodes./*from w w w . j a va 2 s .co m*/ * For each iteration, * for each namenode, * execute a {@link Balancer} to work through all datanodes once. */ static int run(Collection<URI> namenodes, Configuration conf) throws IOException, InterruptedException { final long sleeptime = 2000 * conf.getLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT); LOG.info("namenodes = " + namenodes); // LOG.info("p = " + p); final Formatter formatter = new Formatter(System.out); System.out.println( "Time Stamp Iteration# Bytes Already Moved Bytes Left To Move Bytes Being Moved"); final List<NameNodeConnector> connectors = new ArrayList<NameNodeConnector>(namenodes.size()); try { for (URI uri : namenodes) { connectors.add(new NameNodeConnector(uri, conf)); } boolean done = false; for (int iteration = 0; !done; iteration++) { done = true; Collections.shuffle(connectors); for (NameNodeConnector nnc : connectors) { final Balancer b = new Balancer(nnc, conf); final ReturnStatus r = b.run(iteration, formatter, conf); // clean all lists b.resetData(conf); if (r == ReturnStatus.IN_PROGRESS) { done = false; } else if (r != ReturnStatus.SUCCESS) { //must be an error statue, return. return r.code; } } if (!done) { Thread.sleep(sleeptime); } } } finally { for (NameNodeConnector nnc : connectors) { nnc.close(); } } return ReturnStatus.SUCCESS.code; }
From source file:net.arp7.HdfsPerfTest.WriteFileParameters.java
License:Apache License
/** * Initialize some write parameters from the configuration. * * @param conf/*from w ww . ja va 2s. c om*/ */ private void initDefaultsFromConfiguration(Configuration conf) { blockSize = conf.getLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT); replication = conf.getLong(DFSConfigKeys.DFS_REPLICATION_KEY, DFSConfigKeys.DFS_REPLICATION_DEFAULT); }
From source file:net.jarcec.sqoop.data.gen.mr.GeneratorInputFormat.java
License:Apache License
@Override public List<InputSplit> getSplits(JobContext jobContext) throws IOException, InterruptedException { Configuration configuration = jobContext.getConfiguration(); long files = configuration.getLong(Constants.FILES_COUNT, 0); long records = configuration.getLong(Constants.RECORD_COUNT, 0); List<InputSplit> splits = new LinkedList<InputSplit>(); long next = 1; for (int i = 0; i < files; i++) { splits.add(new GeneratorSplit(next, next + records)); next += records;// w w w. j av a 2 s .com } return splits; }
From source file:net.java.jatextmining.lib.CoOccurrenceWeightingMapper.java
License:Apache License
@Override public void map(Object key, Text value, Context context) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); String type = conf.get("type"); long inputNum = 0; inputNum = conf.getLong("docNum", inputNum); String[] input = value.toString().split("\t"); if (input.length > 2) { String w1 = input[0];/*w ww .j a v a 2s. com*/ String w2 = input[1]; System.err.println("\t" + w1 + "\t" + w2); double a = Double.valueOf(input[2]); double w1Score = 0; double w2Score = 0; if (dfMap.containsKey(w1)) { w1Score = dfMap.get(w1); } if (dfMap.containsKey(w2)) { w2Score = dfMap.get(w2); } if (w1Score != 0 || w2Score != 0) { double b = w2Score - a; double c = w1Score - a; double d = inputNum - (w1Score + w2Score - a); System.err.println("\t" + a + "\t" + b + "\t" + c + "\t" + d); double score = 0.0; if (type.equals("chi")) { double tmp = (a * d) - (b * c); if (tmp == 0) { score = 0; } else { double chiA = inputNum * Math.pow(tmp, 2); double chiB = (a + b) * (a + c) * (c + d) * (b + d); score = chiA / chiB; } System.err.println("\tchi:" + score); } else if (type.equals("mi")) { score = Math.log((a * inputNum) / (a + b) * (a + c)); } else if (type.equals("freaq")) { score = a; } rToken.set(input[0]); rValue.set(input[1] + "\t" + String.valueOf(score)); context.write(rToken, rValue); } } }
From source file:net.thevis.groovyhadoop.backport.CombineFileInputFormat.java
License:Apache License
@Override public List<InputSplit> getSplits(JobContext job) throws IOException { long minSizeNode = 0; long minSizeRack = 0; long maxSize = 0; Configuration conf = job.getConfiguration(); // the values specified by setxxxSplitSize() takes precedence over the // values that might have been specified in the config if (minSplitSizeNode != 0) { minSizeNode = minSplitSizeNode;/* w w w. j a v a 2 s. c o m*/ } else { minSizeNode = conf.getLong(SPLIT_MINSIZE_PERNODE, 0); } if (minSplitSizeRack != 0) { minSizeRack = minSplitSizeRack; } else { minSizeRack = conf.getLong(SPLIT_MINSIZE_PERRACK, 0); } if (maxSplitSize != 0) { maxSize = maxSplitSize; } else { maxSize = conf.getLong("mapreduce.input.fileinputformat.split.maxsize", 0); } if (minSizeNode != 0 && maxSize != 0 && minSizeNode > maxSize) { throw new IOException("Minimum split size pernode " + minSizeNode + " cannot be larger than maximum split size " + maxSize); } if (minSizeRack != 0 && maxSize != 0 && minSizeRack > maxSize) { throw new IOException("Minimum split size per rack" + minSizeRack + " cannot be larger than maximum split size " + maxSize); } if (minSizeRack != 0 && minSizeNode > minSizeRack) { throw new IOException("Minimum split size per node" + minSizeNode + " cannot be smaller than minimum split " + "size per rack " + minSizeRack); } // all the files in input set Path[] paths = FileUtil.stat2Paths(listStatus(job).toArray(new FileStatus[0])); List<InputSplit> splits = new ArrayList<InputSplit>(); if (paths.length == 0) { return splits; } // Convert them to Paths first. This is a costly operation and // we should do it first, otherwise we will incur doing it multiple // times, one time each for each pool in the next loop. List<Path> newpaths = new LinkedList<Path>(); for (int i = 0; i < paths.length; i++) { Path p = new Path(paths[i].toUri().getPath()); newpaths.add(p); } paths = null; // In one single iteration, process all the paths in a single pool. // Processing one pool at a time ensures that a split contains paths // from a single pool only. for (MultiPathFilter onepool : pools) { ArrayList<Path> myPaths = new ArrayList<Path>(); // pick one input path. If it matches all the filters in a pool, // add it to the output set for (Iterator<Path> iter = newpaths.iterator(); iter.hasNext();) { Path p = iter.next(); if (onepool.accept(p)) { myPaths.add(p); // add it to my output set iter.remove(); } } // create splits for all files in this pool. getMoreSplits(conf, myPaths.toArray(new Path[myPaths.size()]), maxSize, minSizeNode, minSizeRack, splits); } // create splits for all files that are not in any pool. getMoreSplits(conf, newpaths.toArray(new Path[newpaths.size()]), maxSize, minSizeNode, minSizeRack, splits); // free up rackToNodes map rackToNodes.clear(); return splits; }
From source file:org.apache.accumulo.core.file.rfile.RFileOperations.java
License:Apache License
@Override protected FileSKVWriter openWriter(OpenWriterOperation options) throws IOException { AccumuloConfiguration acuconf = options.getTableConfiguration(); long blockSize = acuconf.getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE); long indexBlockSize = acuconf.getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX); SamplerConfigurationImpl samplerConfig = SamplerConfigurationImpl.newSamplerConfig(acuconf); Sampler sampler = null;/*from w w w . j a v a2 s.co m*/ if (samplerConfig != null) { sampler = SamplerFactory.newSampler(samplerConfig, acuconf); } String compression = options.getCompression(); compression = compression == null ? options.getTableConfiguration().get(Property.TABLE_FILE_COMPRESSION_TYPE) : compression; FSDataOutputStream outputStream = options.getOutputStream(); Configuration conf = options.getConfiguration(); if (outputStream == null) { int hrep = conf.getInt("dfs.replication", -1); int trep = acuconf.getCount(Property.TABLE_FILE_REPLICATION); int rep = hrep; if (trep > 0 && trep != hrep) { rep = trep; } long hblock = conf.getLong("dfs.block.size", 1 << 26); long tblock = acuconf.getMemoryInBytes(Property.TABLE_FILE_BLOCK_SIZE); long block = hblock; if (tblock > 0) block = tblock; int bufferSize = conf.getInt("io.file.buffer.size", 4096); String file = options.getFilename(); FileSystem fs = options.getFileSystem(); outputStream = fs.create(new Path(file), false, bufferSize, (short) rep, block); } CachableBlockFile.Writer _cbw = new CachableBlockFile.Writer( new RateLimitedOutputStream(outputStream, options.getRateLimiter()), compression, conf, acuconf); RFile.Writer writer = new RFile.Writer(_cbw, (int) blockSize, (int) indexBlockSize, samplerConfig, sampler); return writer; }
From source file:org.apache.accumulo.examples.wikisearch.ingest.WikipediaConfiguration.java
License:Apache License
public static long getMinInputSplitSize(Configuration conf) { return conf.getLong(PARTITIONED_INPUT_MIN_SPLIT_SIZE, 1l << 27); }
From source file:org.apache.accumulo.examples.wikisearch.ingest.WikipediaConfiguration.java
License:Apache License
public static long bulkIngestBufferSize(Configuration conf) { return conf.getLong(BULK_INGEST_BUFFER_SIZE, 1l << 28); }
From source file:org.apache.accumulo.examples.wikisearch.output.SortingRFileOutputFormat.java
License:Apache License
public static long getMaxBufferSize(Configuration conf) { return conf.getLong(MAX_BUFFER_SIZE, -1); }