List of usage examples for org.apache.hadoop.conf Configuration setBoolean
public void setBoolean(String name, boolean value)
name property to a boolean. From source file:com.asakusafw.testdriver.hadoop.ConfigurationFactory.java
License:Apache License
@Override protected void configure(Configuration configuration) { if (preferences.getLocalFileSystemClassName() != null) { configuration.set("fs.file.impl", preferences.getLocalFileSystemClassName()); //$NON-NLS-1$ configuration.setBoolean("fs.fs.impl.disable.cache", true); //$NON-NLS-1$ }/* w ww . j a v a 2 s . c o m*/ }
From source file:com.bah.culvert.data.index.Index.java
License:Apache License
/** * Used to set a key indicating if the string value held by another * configuration key is a base64 encoded binary or not. * @param isValueBinaryEncodedSetting The key telling weather or not the other * key (setting) is base64./*from www.j av a 2s .co m*/ * @param potentiallyEncodedSetting The actual key that might be base64 * encoded. * @param data The data to set as base64. * @param conf The configuration to do the setting on. */ private static void setBinaryConfSetting(String isValueBinaryEncodedSetting, String potentiallyEncodedSetting, byte[] data, Configuration conf) { CharsetDecoder decoder = UTF_8.newDecoder(); decoder.onMalformedInput(CodingErrorAction.REPORT); try { CharBuffer colFamString = decoder.decode(ByteBuffer.wrap(data)); conf.setBoolean(isValueBinaryEncodedSetting, false); conf.set(potentiallyEncodedSetting, colFamString.toString()); } catch (CharacterCodingException e) { conf.setBoolean(isValueBinaryEncodedSetting, true); conf.set(potentiallyEncodedSetting, new String(Base64.encodeBase64(data), UTF_8)); } }
From source file:com.bah.culvert.data.index.TermBasedIndex.java
License:Apache License
/** * Sets the value splitting functionality. If this is set to <tt>false</tt> * then splitting will not occur. To set the regex use * {@link #setTokenRegex(String, Configuration)} * <p>/* w ww. j a v a2 s . c om*/ * If not value is set, then the index will use the default value ( * {@link #DEFAULT_IS_SPLITABLE}). * * @param isSplitable If splitting should occur. * @param conf The configuration to set. */ public static void setSplitable(boolean isSplitable, Configuration conf) { conf.setBoolean(SPLITABLE_TERM_KEY, isSplitable); }
From source file:com.bah.culvert.data.index.TermBasedIndex.java
License:Apache License
/** * Sets whether the index should lower case the terms. * <p>// w w w . j a v a 2 s . c om * If not value is set, then the index will use the default value ( * {@link #DEFAULT_IS_LOWER_CASED}). * * @param isLowerable If terms should be converted to lower case. * @param conf The configuration to set. */ public static void setToLower(boolean isLowerable, Configuration conf) { conf.setBoolean(TO_LOWER_KEY, isLowerable); }
From source file:com.bizosys.hsearch.kv.indexing.KVIndexer.java
License:Apache License
/** * Given a indexing parameters it starts a indexing. * Different indexing type are:// w w w.j a v a2 s . c om * SF2HB = Simple File(csv,tsv) to hbase directly. * SF2HF = Simple File(csv,tsv) to HFile, which can be loaded to Hbase using LoadIncrementalHfiles. class from hbase. * SF2MF = Simple File(csv,tsv) to MapFile (key as {@link Text} and value as {@link BytesWritable}) * MF2HB = Map File(key and value as csv,tsv) to hbase. * MF2HF = Map File(key and value as csv,tsv) to HFile, which can be loaded to Hbase using LoadIncrementalHfiles. class from hbase. * MF2MF = Map File(key and value as csv,tsv) to MapFile(key as {@link Text} and value as {@link BytesWritable}) * HB2HB = Hbase to Hbase * HB2HF = Hbase to HFile which can be loaded to Hbase using LoadIncrementalHfiles. class from hbase. * HB2MF = Hbase to MapFile(key as {@link Text} and value as {@link BytesWritable}) * @param args * @throws IOException * @throws InterruptedException * @throws ClassNotFoundException */ public void execute(String[] args) throws IOException, InterruptedException, ClassNotFoundException { if (args.length < 7) { String err = "Usage : " + KVIndexer.class + " <<Job Type(SF2HB|SF2HF|SF2MF...)>> <<Input Source>> <<Output Sink>> <<XML File Configuration>> <<Skip Header(true|false)>> <<Run KeyGeneration Job>> <<Number Of reducer>> <<Speculative Execution>> <<scanner-cache-size>> <<filter>>"; IdSearchLog.l.fatal(err); System.exit(1); } String msg = this.getClass().getName() + " > Initializing indexer job."; IdSearchLog.l.info(msg); int seq = 0; int len = args.length; String jobType = (len > seq) ? args[seq++] : ""; String inputSource = (len > seq) ? args[seq++] : ""; String outputSink = (len > seq) ? args[seq++] : "/tmp/hsearch-index"; String xmlFilePath = (len > seq) ? args[seq++] : ""; String skipHeader = (len > seq) ? args[seq++] : "false"; boolean runKeyGenJob = (len > seq) ? args[seq++].trim().equalsIgnoreCase("true") : false; int numberOfReducer = (len > seq) ? Integer.parseInt(args[seq++].trim()) : 1; boolean speculativeExecution = (len > seq) ? args[seq++].trim().equalsIgnoreCase("true") : true; int scannerCacheSize = (len > seq) ? Integer.parseInt(args[seq++].trim()) : 300; String filter = (len > seq) ? args[seq++] : ""; if (isEmpty(jobType)) { String err = this.getClass().getName() + " > Please enter Job type as one of these :\n SF2HB|SF2HF|SF2MF|MF2HB|MF2HF|MF2MF|HB2HB|HB2HF|HB2MF|IMF2HF"; System.err.println(err); throw new IOException(err); } if (isEmpty(inputSource)) { String err = this.getClass().getName() + " > Please enter input file path."; System.err.println(err); throw new IOException(err); } Configuration conf = HBaseConfiguration.create(); FieldMapping fm = createFieldMapping(conf, xmlFilePath, new StringBuilder()); outputSink = outputSink.charAt(outputSink.length() - 1) == '/' ? outputSink : outputSink + "/"; outputSink = outputSink + fm.tableName; createHBaseTable(fm); KVIndexer.FAM_NAME = fm.familyName.getBytes(); KVIndexer.FIELD_SEPARATOR = fm.fieldSeparator; conf.set(XML_FILE_PATH, xmlFilePath); conf.set(OUTPUT_FOLDER, outputSink); conf.set(SKIP_HEADER, skipHeader); conf.setBoolean("mapreduce.map.speculative", speculativeExecution); Job job = Job.getInstance(conf, "com.bizosys.hsearch.kv.indexing.KVIndexer type : " + jobType + "\n" + inputSource + "\n" + outputSink); job.setJarByClass(this.getClass()); job.setNumReduceTasks(numberOfReducer); Integer jobTypeI = JobTypeMapping.get(jobType); if (jobTypeI == null) throw new IOException("Invalid Jobtype " + jobType); /** * if internal keyIndex is given then generate the keys first and then do indexing * else just run indexer by creating keys from hbase */ boolean keyGenjobStatus = false; if (-1 != fm.internalKey && runKeyGenJob) { Configuration keyGenConf = HBaseConfiguration.create(); keyGenConf.set(INPUT_SOURCE, inputSource); keyGenConf.set(XML_FILE_PATH, xmlFilePath); keyGenConf.set(OUTPUT_FOLDER, outputSink); keyGenConf.set(SKIP_HEADER, skipHeader); Job keyGenJob = Job.getInstance(keyGenConf, "Creating Keys KVKeyGenerator for " + inputSource); switch (jobTypeI) { case SF2HB: case SF2HF: case SF2MF: { FileInputFormat.addInputPath(keyGenJob, new Path(inputSource)); keyGenJob.setMapperClass(KVKeyGeneratorMapperFile.class); keyGenJob.setInputFormatClass(TextInputFormat.class); keyGenJob.setMapOutputKeyClass(Text.class); keyGenJob.setMapOutputValueClass(Text.class); keyGenJob.setReducerClass(KVKeyGeneratorReducerFile.class); keyGenJob.setNumReduceTasks(numberOfReducer); keyGenJob.setOutputKeyClass(NullWritable.class); keyGenJob.setOutputValueClass(Text.class); inputSource = outputSink + "_" + INPUTWITH_KEY; Path intermediatePath = new Path(inputSource); System.out.println("Final input path " + inputSource); FileOutputFormat.setOutputPath(keyGenJob, intermediatePath); keyGenjobStatus = keyGenJob.waitForCompletion(true); if (!keyGenjobStatus) { throw new IOException("Error in running Job for Key Generation"); } break; } case HB2HB: case HB2HF: case HB2MF: { Scan scan = new Scan(); scan.setCaching(scannerCacheSize); scan.setCacheBlocks(false); // Added Filter if (null != filter) { if (filter.trim().length() > 0) { int index = filter.indexOf('='); scan.setFilter(new SingleColumnValueFilter(fm.familyName.getBytes(), filter.substring(0, index).getBytes(), CompareOp.EQUAL, filter.substring(index + 1).getBytes())); } } byte[] family = fm.familyName.getBytes(); for (String name : fm.nameWithField.keySet()) { Field fld = fm.nameWithField.get(name); if (!fld.isMergedKey) continue; scan.addColumn(family, fld.sourceName.trim().getBytes()); } TableMapReduceUtil.initTableMapperJob(inputSource, // input table scan, // Scan instance to control CF and attribute selection KVKeyGeneratorMapperHBase.class, // mapper class Text.class, // mapper output key ImmutableBytesWritable.class, // mapper output value keyGenJob); TableMapReduceUtil.initTableReducerJob(inputSource, // output table KVKeyGeneratorReducerHBase.class, // reducer class keyGenJob); keyGenjobStatus = keyGenJob.waitForCompletion(true); if (!keyGenjobStatus) { throw new IOException("Error in running Job for Key Generation"); } break; } case MF2HB: case MF2HF: case MF2MF: { break; } default: break; } } /* * Run job based on job type eg. SF2HB,SF2MF,SF2HF etc. */ System.out.println("Sending path " + inputSource); runJob(jobTypeI, job, fm, inputSource, outputSink, scannerCacheSize, filter); }
From source file:com.blackberry.bdp.kaboom.StartupConfig.java
License:Apache License
/** * Instantiates properties from either the specified configuration file or the default for class * * @return Configuration/* w w w. j a va2 s .co m*/ */ private Configuration buildHadoopConfiguration() throws FileNotFoundException { Configuration newHadoopConfiguration = new Configuration(); if (new File("/etc/hadoop/conf/core-site.xml").exists()) { newHadoopConfiguration.addResource(new FileInputStream("/etc/hadoop/conf/core-site.xml")); } else { LOG.warn("/etc/hadoop/conf/core-site.xml does not exist or cannot be read"); } if (new File("/etc/hadoop/conf/hdfs-site.xml").exists()) { newHadoopConfiguration.addResource(new FileInputStream("/etc/hadoop/conf/hdfs-site.xml")); } else { LOG.warn("/etc/hadoop/conf/hdfs-site.xml does not exist or cannot be read"); } // Adds any more standard configs we find in the classpath for (String file : new String[] { "core-site.xml", "hdfs-site.xml" }) { InputStream in = this.getClass().getClassLoader().getResourceAsStream(file); if (in != null) { newHadoopConfiguration.addResource(in); } } newHadoopConfiguration.setBoolean("fs.automatic.close", false); return newHadoopConfiguration; }
From source file:com.blackberry.logtools.LogTools.java
License:Apache License
public void runMRJob(boolean quiet, boolean silent, Configuration conf, ArrayList<String> D_options, String out, Logger LOG, String field_separator, String queue_name, String[] args, String job, Tool tool) throws Exception { logConsole(quiet, silent, info, "Running Mapreduce job & Calling " + job); if (out.equals("-")) { //Uncompress results to be able to read to stdout D_options.add("-Dmapreduce.output.fileoutputformat.compress=false"); }/*from www .j av a2s . co m*/ try { conf.set("zk.connect.string", System.getenv("ZK_CONNECT_STRING")); conf.setBoolean("mapreduce.output.fileoutputformat.compress", true); conf.set("mapred.output.compress.codec", "org.apache.hadoop.io.compress.SnappyCodec"); conf.setInt("mapred.max.split.size", 256 * 1024 * 1024); conf.set("logdriver.output.field.separator", field_separator); conf.set("mapred.job.queue.name", StringEscapeUtils.escapeJava(queue_name)); dOpts(D_options, silent, out, conf); //Now run JOB and send arguments LOG.info("Sending args to " + job + ": {}", args); ToolRunner.run(conf, tool, args); } catch (IOException e) { if (e.toString().contains("Failed to find any Kerberos")) { logConsole(true, true, error, "No/bad Kerberos ticket - please authenticate."); System.exit(1); } else if (e.toString().contains("Permission denied")) { logConsole(true, true, error, "Permission denied."); System.err.println("; Please go to https://go/itforms and filled out the Hadoop Onboarding Form " + "to get access to the requested data. Paste the following data into the ticket to help with your request:\n" + "Error Message" + e); System.exit(1); } else if (e.toString().contains("quota") && e.toString().contains("exceeded")) { logConsole(true, true, error, "Disk quota Exceeded."); System.exit(1); } logConsole(true, true, error, "\n\tError running mapreduce job." + generalError() + "\n\tCommand stopped"); e.printStackTrace(); System.exit(1); } }
From source file:com.cloudera.castagna.logparser.mr.StatusCodesStats.java
License:Apache License
@Override public int run(String[] args) throws Exception { if (args.length != 2) { System.err.printf("Usage: %s [generic options] <input> <output>\n", getClass().getName()); ToolRunner.printGenericCommandUsage(System.err); return -1; }//w ww. ja va2s . c o m Configuration configuration = getConf(); boolean useCompression = configuration.getBoolean(Constants.OPTION_USE_COMPRESSION, Constants.OPTION_USE_COMPRESSION_DEFAULT); if (useCompression) { configuration.setBoolean("mapred.compress.map.output", true); configuration.set("mapred.output.compression.type", "BLOCK"); configuration.set("mapred.map.output.compression.codec", "org.apache.hadoop.io.compress.GzipCodec"); } boolean overrideOutput = configuration.getBoolean(Constants.OPTION_OVERWRITE_OUTPUT, Constants.OPTION_OVERWRITE_OUTPUT_DEFAULT); FileSystem fs = FileSystem.get(new Path(args[1]).toUri(), configuration); if (overrideOutput) { fs.delete(new Path(args[1]), true); } Job job = Job.getInstance(configuration); job.setJobName(Constants.STATUS_CODES_STATS); job.setJarByClass(getClass()); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); job.setInputFormatClass(TextInputFormat.class); job.setMapperClass(StatusCodesStatsMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); job.setCombinerClass(StatusCodesStatsCombiner.class); job.setReducerClass(StatusCodesStatsReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); Utils.setReducers(job, configuration, log); job.setOutputFormatClass(TextOutputFormat.class); if (log.isDebugEnabled()) Utils.log(job, log); return job.waitForCompletion(true) ? 0 : 1; }
From source file:com.cloudera.HdfsClientHeapBench.java
License:Apache License
public static void main(String[] args) throws Exception { System.out.println("running HdfsClientHeapBench: benchmarks " + "input stream size in Hadoop...\n"); final int NUM_OPENS = 50000; if (args.length < 1) { System.err.println("You must specify a single argument: the URI " + "of a directory to test.\n" + "Examples: file:///tmp, hdfs:///\n"); System.exit(1);/*from w w w . j a v a 2s .com*/ } final String uri = args[0]; Configuration conf = new Configuration(); conf.setBoolean("dfs.client.read.shortcircuit", false); FSDataInputStream[] streams = new FSDataInputStream[NUM_OPENS]; try { FileSystem dfs = FileSystem.get(new URI(uri), conf); final Path TEST_PATH = new Path("/testFile"); createFile(dfs, TEST_PATH, 131072); for (int i = 0; i < NUM_OPENS; i++) { streams[i] = dfs.open(TEST_PATH); System.out.println("opening file " + i + "..."); if (0 != streams[i].read()) { throw new IOException("failed to read a byte from stream " + i + ": unexpected EOF."); } streams[i].unbuffer(); } // Sleep for a long time so we can run jmat to get a heap dump Thread.sleep(9000000L); } finally { for (FSDataInputStream stream : streams) { try { if (stream != null) { stream.close(); } } catch (IOException e) { System.out.println("error closing stream: " + e.getMessage()); } } } }
From source file:com.cloudera.llama.am.impl.TestExpansionReservationsLlamaAM.java
License:Apache License
@Test public void testReleaseReservationWithExpansionMultipleClients() throws Exception { Configuration conf = new Configuration(false); conf.setClass(LlamaAM.RM_CONNECTOR_CLASS_KEY, RecordingMockRMConnector.class, RMConnector.class); conf.setBoolean(LlamaAM.NORMALIZING_ENABLED_KEY, false); conf.setBoolean(LlamaAM.CACHING_ENABLED_KEY, false); SingleQueueLlamaAM am = new SingleQueueLlamaAM(conf, "queue", Executors.newScheduledThreadPool(4)); am.setCallback(new DummySingleQueueLlamaAMCallback()); ExpansionReservationsLlamaAM eAm = new ExpansionReservationsLlamaAM(am); try {/*w w w.j a v a 2 s .com*/ eAm.start(); Reservation r = TestUtils.createReservation(true); PlacedReservation pr = TestUtils.createPlacedReservation(r, PlacedReservation.Status.ALLOCATED); eAm.reserve(pr.getReservationId(), r); UUID resource1Id = pr.getPlacedResources().get(0).getResourceId(); RMEvent change = RMEvent.createStatusChangeEvent(resource1Id, PlacedResource.Status.ALLOCATED); am.onEvent(Arrays.asList(change)); Expansion e1 = TestUtils.createExpansion(pr); UUID eId1 = eAm.expand(e1); Assert.assertNotNull(eId1); Assert.assertEquals(new HashSet<UUID>(Arrays.asList(eId1)), eAm.getExpansions(pr.getReservationId())); Expansion e2 = TestUtils.createExpansion(pr); UUID eId2 = eAm.expand(e2); Assert.assertNotNull(eId2); Assert.assertEquals(new HashSet<UUID>(Arrays.asList(eId1, eId2)), eAm.getExpansions(pr.getReservationId())); eAm.releaseReservation(pr.getHandle(), pr.getReservationId(), false); Assert.assertNull(eAm.getExpansions(pr.getReservationId())); } finally { eAm.stop(); } }