List of usage examples for org.apache.hadoop.conf Configuration setInt
public void setInt(String name, int value)
name property to an int. From source file:com.hortonworks.minicluster.MiniHadoopCluster.java
License:Apache License
/** */*from w w w . j a v a 2 s . co m*/ */ @Override public void serviceInit(Configuration conf) throws Exception { conf.setBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, true); conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] { ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID }); conf.setClass( String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID), ShuffleHandler.class, Service.class); conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0); this.addService(new ResourceManagerWrapper()); for (int index = 0; index < this.nodeManagers.length; index++) { this.nodeManagers[index] = new ShortCircuitedNodeManager(); this.addService(new NodeManagerWrapper(index)); } super.serviceInit(conf instanceof YarnConfiguration ? conf : new YarnConfiguration(conf)); }
From source file:com.ikanow.infinit.e.data_model.custom.InfiniteMongoConfigUtil.java
License:Apache License
public static void setMaxSplits(Configuration conf, int maxSplits) { conf.setInt(MAX_SPLITS, maxSplits); }
From source file:com.ikanow.infinit.e.data_model.custom.InfiniteMongoConfigUtil.java
License:Apache License
public static void setMaxDocsPerSplit(Configuration conf, int maxDocsPerSplit) { conf.setInt(MAX_DOCS_PER_SPLIT, maxDocsPerSplit); }
From source file:com.intel.hadoop.hbase.dot.StringRange.java
License:Apache License
private static void initialize(Configuration conf) { conf.setInt("hbase.client.retries.number", 1); try {/*from w ww.j a va2 s . c o m*/ admin = TEST_UTIL.getHBaseAdmin(); } catch (IOException e) { assertNull("Failed to connect to HBase master", e); } assertNotNull("No HBaseAdmin instance created", admin); try { createTestTableForFilters(); } catch (IOException e) { assertNull("Failed to create Test Table for Filters", e); } }
From source file:com.jumptap.h2redis.RedisDriver.java
License:Open Source License
@Override public int run(String[] args) throws Exception { if (args.length < 5) { usage();/*from ww w. jav a2 s.com*/ return 1; } Map<String, String> argMap = new HashMap<String, String>(); String[] kv; for (String arg : args) { kv = arg.split("="); if (kv.length != 2) { usage(); return 1; } argMap.put(kv[0].trim(), kv[1]); } Configuration conf = getConf(); String[] hostPort = argMap.get(REDIS_CMD).split(":"); conf.set(REDIS_HOST, hostPort[0].trim()); conf.setInt(REDIS_PORT, Integer.valueOf(hostPort[1].trim())); conf.setInt(REDIS_KEY_FIELD, Integer.valueOf(argMap.get(KEY_CMD).trim())); conf.setInt(REDIS_HASHKEY_FIELD, Integer.valueOf(argMap.get(HASH_KEY_CMD).trim())); conf.setInt(REDIS_HASHVAL_FIELD, Integer.valueOf(argMap.get(HASH_VAL_CMD).trim())); if (argMap.containsKey(REDIS_DB_CMD)) { conf.set(REDIS_DB, argMap.get(REDIS_DB_CMD).trim()); } if (argMap.containsKey(REDIS_PW_CMD)) { conf.set(REDIS_PW, argMap.get(REDIS_PW_CMD).trim()); } if (argMap.containsKey(KEY_PFX_CMD)) { conf.set(REDIS_KEY_PREFIX, argMap.get(KEY_PFX_CMD).trim()); } if (argMap.containsKey(HASH_KEY_PFX_CMD)) { conf.set(REDIS_HASHKEY_PREFIX, argMap.get(HASH_KEY_PFX_CMD).trim()); } if (argMap.containsKey(KEY_PFX_DELIM_CMD)) { conf.set(REDIS_KEY_PREFIX_DELIM, argMap.get(KEY_PFX_DELIM_CMD).trim()); } if (argMap.containsKey(KEY_FILTER_CMD)) { conf.setPattern(REDIS_KEY_FILTER, Pattern.compile(argMap.get(KEY_FILTER_CMD).trim())); } if (argMap.containsKey(HASH_FILTER_CMD)) { conf.setPattern(REDIS_HASH_FILTER, Pattern.compile(argMap.get(HASH_FILTER_CMD).trim())); } if (argMap.containsKey(VAL_FILTER_CMD)) { conf.setPattern(REDIS_VAL_FILTER, Pattern.compile(argMap.get(VAL_FILTER_CMD).trim())); } if (argMap.containsKey(VAL_FILTER_CMD)) { conf.setPattern(REDIS_VAL_FILTER, Pattern.compile(argMap.get(VAL_FILTER_CMD).trim())); } if (argMap.containsKey(TTL_CMD)) { conf.setInt(REDIS_KEY_TTL, Integer.valueOf(argMap.get(TTL_CMD).trim())); } if (argMap.containsKey(TS_KEY_CMD)) { conf.set(REDIS_KEY_TS, argMap.get(TS_KEY_CMD).trim()); } else { conf.set(REDIS_KEY_TS, "redis.lastupdate"); } Job job = new Job(conf, "RedisDriver"); FileInputFormat.addInputPath(job, new Path(argMap.get(INPUT_CMD))); job.setJarByClass(RedisDriver.class); job.setMapperClass(RedisOutputMapper.class); job.setNumReduceTasks(0); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(RedisOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); return job.waitForCompletion(true) ? 0 : 1; }
From source file:com.koda.integ.hbase.test.OffHeapBlockCacheMultiPerfTest.java
License:Open Source License
/** * Sets the up./* w ww . jav a 2 s. com*/ * * @throws Exception the exception */ protected static void setUp() throws Exception { Configuration config = new Configuration(); // Set L2 config config.set(OffHeapBlockCache.BLOCK_CACHE_MEMORY_SIZE, Long.toString(sRAMCacheSize)); config.setBoolean(OffHeapBlockCache.BLOCK_CACHE_OVERFLOW_TO_EXT_STORAGE_ENABLED, true); config.setLong(OffHeapBlockCache.BLOCK_CACHE_EXT_STORAGE_MEMORY_SIZE, (long) (sDiskMetaRatio * sRAMCacheSize)); config.set(OffHeapBlockCache.BLOCK_CACHE_COMPRESSION, "LZ4"); config.setBoolean(OffHeapBlockCache.BLOCK_CACHE_PERSISTENT, sIsPersistent); config.set(OffHeapBlockCache.BLOCK_CACHE_DATA_ROOTS, sSystemDataDir); // Set L3 config config.set(FileExtStorage.FILE_STORAGE_BASE_DIR, baseDir); // 120G config.set(FileExtStorage.FILE_STORAGE_MAX_SIZE, Long.toString(sDiskCacheSize)); config.set(OffHeapBlockCache.BLOCK_CACHE_EXT_STORAGE_IMPL, "com.koda.integ.hbase.storage.FileExtMultiStorage"); // 2G file size limit config.setLong(FileExtStorage.FILE_STORAGE_FILE_SIZE_LIMIT, sMaxFileSize); // 8MB buffer size config.setInt(FileExtStorage.FILE_STORAGE_BUFFER_SIZE, 8 * 1024 * 1024); config.setInt(FileExtStorage.FILE_STORAGE_NUM_BUFFERS, 2); if (sIsPersistent == false) { //checkDir(); deleteData(); } else { // Set deserializer CacheableSerializer.setSerializer(ByteArrayCacheable.deserializer); } // Create block cache sCache = new OffHeapBlockCache(config); sStorage = (FileExtMultiStorage) sCache.getExternalStorage(); }
From source file:com.koda.integ.hbase.test.OffHeapBlockCachePerfTest.java
License:Open Source License
/** * Sets the up.// w w w. j a v a 2 s. com * * @throws Exception the exception */ protected static void setUp() throws Exception { Configuration config = new Configuration(); // Set L2 config config.set(OffHeapBlockCache.BLOCK_CACHE_MEMORY_SIZE, Long.toString(sRAMCacheSize)); config.setBoolean(OffHeapBlockCache.BLOCK_CACHE_OVERFLOW_TO_EXT_STORAGE_ENABLED, true); config.setLong(OffHeapBlockCache.BLOCK_CACHE_EXT_STORAGE_MEMORY_SIZE, (long) (sDiskMetaRatio * sRAMCacheSize)); config.set(OffHeapBlockCache.BLOCK_CACHE_COMPRESSION, "LZ4"); config.setBoolean(OffHeapBlockCache.BLOCK_CACHE_PERSISTENT, sIsPersistent); config.set(OffHeapBlockCache.BLOCK_CACHE_DATA_ROOTS, sSystemDataDir); // Set L3 config config.set(FileExtStorage.FILE_STORAGE_BASE_DIR, baseDir); // 120G config.set(FileExtStorage.FILE_STORAGE_MAX_SIZE, Long.toString(sDiskCacheSize)); config.set(OffHeapBlockCache.BLOCK_CACHE_EXT_STORAGE_IMPL, "com.koda.integ.hbase.storage.FileExtStorage"); // 2G file size limit config.setLong(FileExtStorage.FILE_STORAGE_FILE_SIZE_LIMIT, sMaxFileSize); // 8MB buffer size config.setInt(FileExtStorage.FILE_STORAGE_BUFFER_SIZE, 8 * 1024 * 1024); config.setInt(FileExtStorage.FILE_STORAGE_NUM_BUFFERS, 2); if (sIsPersistent == false) { checkDir(); } else { // Set deserializer CacheableSerializer.setSerializer(ByteArrayCacheable.deserializer); } // Create block cache sCache = new OffHeapBlockCache(config); sStorage = (FileExtStorage) sCache.getExternalStorage(); }
From source file:com.kse.bigdata.main.Driver.java
License:Apache License
public static void main(String[] args) throws Exception { /********************************************************************************** ** Merge the source files into one. ** /** Should change the directories of each file before executing the program ** ***********************************************************************************/ // String inputFileDirectory = "/media/bk/??/BigData_Term_Project/Debug"; // String resultFileDirectory = "/media/bk/??/BigData_Term_Project/debug.csv"; // File resultFile = new File(resultFileDirectory); // if(!resultFile.exists()) // new SourceFileMerger(inputFileDirectory, resultFileDirectory).mergeFiles(); /********************************************************************************** * Hadoop Operation./*from w w w. jav a 2s. c o m*/ * Befort Start, Check the Length of Sequence We Want to Predict. **********************************************************************************/ Configuration conf = new Configuration(); //Enable MapReduce intermediate compression as Snappy conf.setBoolean("mapred.compress.map.output", true); conf.set("mapred.map.output.compression.codec", "org.apache.hadoop.io.compress.SnappyCodec"); //Enable Profiling //conf.setBoolean("mapred.task.profile", true); String testPath = null; String inputPath = null; String outputPath = null; int sampleSize = 1; ArrayList<String> results = new ArrayList<String>(); for (int index = 0; index < args.length; index++) { /* * Mandatory command */ //Extract input path string from command line. if (args[index].equals("-in")) inputPath = args[index + 1]; //Extract output path string from command line. if (args[index].equals("-out")) outputPath = args[index + 1]; //Extract test data path string from command line. if (args[index].equals("-test")) testPath = args[index + 1]; /* * Optional command */ //Extract a number of neighbors. if (args[index].equals("-nn")) conf.setInt(Reduce.NUMBER_OF_NEAREAST_NEIGHBOR, Integer.parseInt(args[index + 1])); //Whether job uses normalization or not. if (args[index].equals("-norm")) conf.setBoolean(Map.NORMALIZATION, true); //Extract the number of sample size to test. if (args[index].equals("-s")) sampleSize = Integer.valueOf(args[index + 1]); //Whether job uses mean or median //[Default : mean] if (args[index].equals("-med")) conf.setBoolean(Reduce.MEDIAN, true); } String outputFileName = "part-r-00000"; SequenceSampler sampler = new SequenceSampler(testPath, sampleSize); LinkedList<Sequence> testSequences = sampler.getRandomSample(); // Test Sequence // String testSeqString = "13.591-13.674-13.778-13.892-13.958-14.049-14.153-14.185-14.169-14.092-13.905-13.702-13.438-13.187-13.0-12.914-12.868-12.766-12.62-12.433-12.279-12.142-12.063-12.025-100"; // Sequence testSeq = new Sequence(testSeqString); // LinkedList<Sequence> testSequences = new LinkedList<>(); // testSequences.add(testSeq); for (Sequence seq : testSequences) { /* ******************** Hadoop Launch *********************** */ System.out.println(seq.getTailString()); conf.set(Map.INPUT_SEQUENCE, seq.toString()); Job job = new Job(conf); job.setJarByClass(Driver.class); job.setJobName("term-project-driver"); job.setMapperClass(Map.class); job.setMapOutputKeyClass(NullWritable.class); job.setMapOutputValueClass(Text.class); // Should think another way to implement the combiner class // Current Implementation is not helpful to Job. // job.setCombinerClass(Combiner.class); //Set 1 for number of reduce task for keeping 100 most neighbors in sorted set. job.setNumReduceTasks(1); job.setReducerClass(Reduce.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); FileInputFormat.setInputPaths(job, new Path(inputPath)); FileOutputFormat.setOutputPath(job, new Path(outputPath)); job.waitForCompletion(true); /* * if job finishes, get result of the job and store it in results(list). */ try { FileSystem hdfs = FileSystem.get(new Configuration()); BufferedReader fileReader = new BufferedReader( new InputStreamReader(hdfs.open(new Path(outputPath + "/" + outputFileName)))); String line; while ((line = fileReader.readLine()) != null) { results.add(seq.getSeqString() + " " + line); } fileReader.close(); hdfs.delete(new Path(outputPath), true); hdfs.close(); } catch (IOException e) { e.printStackTrace(); System.exit(1); } } /* * if all jobs finish, store results of jobs to output/result.txt file. */ String finalOutputPath = "output/result.csv"; try { FileSystem hdfs = FileSystem.get(new Configuration()); Path file = new Path(finalOutputPath); if (hdfs.exists(file)) { hdfs.delete(file, true); } OutputStream os = hdfs.create(file); PrintWriter printWriter = new PrintWriter(new OutputStreamWriter(os, "UTF-8")); //CSV File Header printWriter.println("Actual,Predicted,MER,MAE"); printWriter.flush(); for (String result : results) { String[] tokens = result.split("\\s+"); printWriter.println(tokens[0] + "," + tokens[1] + "," + tokens[2] + "," + tokens[3]); printWriter.flush(); } printWriter.close(); hdfs.close(); } catch (IOException e) { e.printStackTrace(); System.exit(1); } }
From source file:com.kylinolap.job.hadoop.cube.CuboidJob.java
License:Apache License
protected void setReduceTaskNum(Job job, KylinConfig config, String cubeName, int level) throws ClassNotFoundException, IOException, InterruptedException, JobException { Configuration jobConf = job.getConfiguration(); KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); CubeDesc cubeDesc = CubeManager.getInstance(config).getCube(cubeName).getDescriptor(); double perReduceInputMB = kylinConfig.getDefaultHadoopJobReducerInputMB(); double reduceCountRatio = kylinConfig.getDefaultHadoopJobReducerCountRatio(); // total map input MB double totalMapInputMB = this.getTotalMapInputMB(); // output / input ratio int preLevelCuboids, thisLevelCuboids; if (level == 0) { // base cuboid preLevelCuboids = thisLevelCuboids = 1; } else { // n-cuboid int[] allLevelCount = CuboidCLI.calculateAllLevelCount(cubeDesc); preLevelCuboids = allLevelCount[level - 1]; thisLevelCuboids = allLevelCount[level]; }//from ww w .j av a 2 s .com // total reduce input MB double totalReduceInputMB = totalMapInputMB * thisLevelCuboids / preLevelCuboids; // number of reduce tasks int numReduceTasks = (int) Math.round(totalReduceInputMB / perReduceInputMB * reduceCountRatio); // adjust reducer number for cube which has DISTINCT_COUNT measures for // better performance if (cubeDesc.hasHolisticCountDistinctMeasures()) { numReduceTasks = numReduceTasks * 4; } // at least 1 reducer numReduceTasks = Math.max(1, numReduceTasks); // no more than 5000 reducer by default numReduceTasks = Math.min(kylinConfig.getHadoopJobMaxReducerNumber(), numReduceTasks); jobConf.setInt(MAPRED_REDUCE_TASKS, numReduceTasks); System.out.println("Having total map input MB " + Math.round(totalMapInputMB)); System.out.println("Having level " + level + ", pre-level cuboids " + preLevelCuboids + ", this level cuboids " + thisLevelCuboids); System.out.println("Having per reduce MB " + perReduceInputMB + ", reduce count ratio " + reduceCountRatio); System.out.println("Setting " + MAPRED_REDUCE_TASKS + "=" + numReduceTasks); }
From source file:com.lightboxtechnologies.spectrum.BlockHasher.java
License:Apache License
public int run(String[] args) throws Exception { if (args.length != 3) { System.err.println("Usage: BlockHasher <imageID> <image> <output>"); return 2; }/*from w ww. j av a2 s . c o m*/ final String imageID = args[0]; final String image = args[1]; final String output = args[2]; Configuration conf = getConf(); final Job job = SKJobFactory.createJobFromConf(imageID, image, "BlockHasher", conf); job.setJarByClass(BlockHasher.class); job.setMapperClass(BlockHashMapper.class); // job.setReducerClass(Reducer.class); job.setNumReduceTasks(0); // job ctor copies the Configuration we pass it, get the real one conf = job.getConfiguration(); conf.setLong("timestamp", System.currentTimeMillis()); job.setInputFormatClass(RawFileInputFormat.class); RawFileInputFormat.addInputPath(job, new Path(image)); job.setOutputFormatClass(TextOutputFormat.class); job.setOutputKeyClass(LongWritable.class); job.setOutputValueClass(MD5Hash.class); FileOutputFormat.setOutputPath(job, new Path(output)); conf.setInt("mapred.job.reuse.jvm.num.tasks", -1); return job.waitForCompletion(true) ? 0 : 1; }