List of usage examples for org.apache.hadoop.conf Configuration dumpConfiguration
public static void dumpConfiguration(Configuration config, Writer out) throws IOException
{ "properties" : [ { key : "key1", value : "value1", isFinal : "key1.isFinal", resource : "key1.resource" }, { key : "key2", value : "value2", isFinal : "ke2.isFinal", resource : "key2.resource" } ] } It does not output the properties of the configuration object which is loaded from an input stream. From source file:com.asiainfo.srd.HioBench.java
License:Apache License
public static void main(String[] args) throws Exception { options = new Options(); final Configuration conf = new Configuration(); if (options.dumpConf) { Configuration.dumpConfiguration(conf, new PrintWriter(System.out)); }//from w ww. j av a2 s.c o m final FileSystem fs = FileSystem.get(new URI(options.hdfsUri), conf); if (!fs.exists(options.filePath)) { System.out.println("no file at " + options.filePath + "; writing " + "new file now with length " + options.nGigsInFile + " gigs..."); writeFile(fs); System.out.println("done."); } else if (fs.getLength(options.filePath) != options.nBytesInFile) { System.out.println("existing file " + options.filename + " has length " + fs.getLength(options.filePath) + ", but we wanted length " + options.nBytesInFile + ". Re-creating."); writeFile(fs); System.out.println("done."); } else { System.out.println( "using existing file at " + options.filePath + " of length " + options.nGigsInFile + " gigs."); } long nanoStart = System.nanoTime(); WorkerThread threads[] = new WorkerThread[options.nThreads]; for (int i = 0; i < options.nThreads; i++) { threads[i] = new WorkerThread(i == 0, fs, WorkerThread.createBenchReader(options, i)); } for (int i = 0; i < options.nThreads; i++) { threads[i].start(); } for (int i = 0; i < options.nThreads; i++) { threads[i].join(); } for (int i = 0; i < options.nThreads; i++) { Throwable t = threads[i].getException(); if (t != null) { System.err.println("there were exceptions. Aborting."); System.exit(1); } } long nanoEnd = System.nanoTime(); fs.close(); long totalIo = options.nThreads; totalIo *= options.nBytesToRead; float nanoDiff = nanoEnd - nanoStart; float seconds = nanoDiff / 1000000000; System.out.println(String.format("Using %d threads, read %s in %f seconds", options.nThreads, prettyPrintByteSize(totalIo), seconds)); float rate = totalIo / seconds; System.out.println("Average rate was " + prettyPrintByteSize(rate) + "/s"); }
From source file:com.cloudera.HioBench.java
License:Apache License
public static void main(String[] args) throws Exception { options = new Options(); final Configuration conf = new Configuration(); if (options.dumpConf) { Configuration.dumpConfiguration(conf, new PrintWriter(System.out)); }//from w w w . j a va 2 s . c o m final FileSystem fs = FileSystem.get(new URI(options.hdfsUri), conf); fs.setVerifyChecksum(!options.skipChecksum); if (!fs.exists(options.filePath)) { System.out.println("no file at " + options.filePath + "; writing " + "new file now with length " + options.nGigsInFile + " gigs..."); writeFile(fs); System.out.println("done."); } else if (fs.getLength(options.filePath) != options.nBytesInFile) { System.out.println("existing file " + options.filename + " has length " + fs.getLength(options.filePath) + ", but we wanted length " + options.nBytesInFile + ". Re-creating."); writeFile(fs); System.out.println("done."); } else { System.out.println( "using existing file at " + options.filePath + " of length " + options.nGigsInFile + " gigs."); } long nanoStart = System.nanoTime(); WorkerThread threads[] = new WorkerThread[options.nThreads]; for (int i = 0; i < options.nThreads; i++) { threads[i] = new WorkerThread(i == 0, fs, WorkerThread.createBenchReader(options, i)); } for (int i = 0; i < options.nThreads; i++) { threads[i].start(); } for (int i = 0; i < options.nThreads; i++) { threads[i].join(); } for (int i = 0; i < options.nThreads; i++) { Throwable t = threads[i].getException(); if (t != null) { System.err.println("there were exceptions. Aborting."); System.exit(1); } } long nanoEnd = System.nanoTime(); fs.close(); long totalIo = options.nThreads; totalIo *= options.nBytesToRead; float nanoDiff = nanoEnd - nanoStart; float seconds = nanoDiff / 1000000000; System.out.println(String.format("Using %d threads, read %s in %f seconds", options.nThreads, prettyPrintByteSize(totalIo), seconds)); float rate = totalIo / seconds; System.out.println("Average rate was " + prettyPrintByteSize(rate) + "/s"); }
From source file:com.datatorrent.stram.StreamingAppMasterService.java
License:Apache License
/** * Dump out contents of $CWD and the environment to stdout for debugging *///from w w w.j a v a 2s . c o m @SuppressWarnings("UseOfSystemOutOrSystemErr") public void dumpOutDebugInfo() { LOG.info("Dump debug output"); Map<String, String> envs = System.getenv(); LOG.info("\nDumping System Env: begin"); for (Map.Entry<String, String> env : envs.entrySet()) { LOG.info("System env: key=" + env.getKey() + ", val=" + env.getValue()); } LOG.info("Dumping System Env: end"); String cmd = "ls -al"; Runtime run = Runtime.getRuntime(); Process pr; try { pr = run.exec(cmd); pr.waitFor(); BufferedReader buf = new BufferedReader(new InputStreamReader(pr.getInputStream())); String line; LOG.info("\nDumping files in local dir: begin"); try { while ((line = buf.readLine()) != null) { LOG.info("System CWD content: " + line); } LOG.info("Dumping files in local dir: end"); } finally { buf.close(); } } catch (IOException e) { LOG.debug("Exception", e); } catch (InterruptedException e) { LOG.info("Interrupted", e); } LOG.info("Classpath: {}", System.getProperty("java.class.path")); LOG.info("Config resources: {}", getConfig().toString()); try { // find a better way of logging this using the logger. Configuration.dumpConfiguration(getConfig(), new PrintWriter(System.out)); } catch (Exception e) { LOG.error("Error dumping configuration.", e); } }
From source file:org.apache.falcon.workflow.util.OozieActionConfigurationHelper.java
License:Apache License
public static void dumpConf(Configuration conf, String message) throws IOException { StringWriter writer = new StringWriter(); Configuration.dumpConfiguration(conf, writer); LOG.info(message + " {}", writer); }
From source file:org.apache.hive.http.ConfServlet.java
License:Apache License
/** * Guts of the servlet - extracted for easy testing. *//* w w w . jav a 2 s .c o m*/ static void writeResponse(Configuration conf, Writer out, String format) throws IOException, BadFormatException { if (FORMAT_JSON.equals(format)) { Configuration.dumpConfiguration(conf, out); } else if (FORMAT_XML.equals(format)) { conf.writeXml(out); } else { throw new BadFormatException("Bad format: " + format); } }
From source file:org.apache.oozie.action.hadoop.LauncherMapper.java
License:Apache License
/** * Pushing all important conf to hadoop conf for the action *///from w w w. ja v a 2 s .co m private void propagateToHadoopConf() throws IOException { Configuration propagationConf = new Configuration(false); if (System.getProperty(OOZIE_ACTION_ID) != null) { propagationConf.set(OOZIE_ACTION_ID, System.getProperty(OOZIE_ACTION_ID)); } if (System.getProperty(OOZIE_JOB_ID) != null) { propagationConf.set(OOZIE_JOB_ID, System.getProperty(OOZIE_JOB_ID)); } if (System.getProperty(OOZIE_LAUNCHER_JOB_ID) != null) { propagationConf.set(OOZIE_LAUNCHER_JOB_ID, System.getProperty(OOZIE_LAUNCHER_JOB_ID)); } // loading action conf prepared by Oozie Configuration actionConf = LauncherMain.loadActionConf(); if (actionConf.get(LauncherMainHadoopUtils.CHILD_MAPREDUCE_JOB_TAGS) != null) { propagationConf.set(LauncherMain.MAPREDUCE_JOB_TAGS, actionConf.get(LauncherMainHadoopUtils.CHILD_MAPREDUCE_JOB_TAGS)); } propagationConf.writeXml(new FileWriter(PROPAGATION_CONF_XML)); Configuration.dumpConfiguration(propagationConf, new OutputStreamWriter(System.out)); Configuration.addDefaultResource(PROPAGATION_CONF_XML); }
From source file:org.apache.sentry.api.service.thrift.ConfServlet.java
License:Apache License
@Override public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { String format = request.getParameter(FORMAT_PARAM); if (format == null) { format = FORMAT_XML;//from w ww . ja v a 2s. c o m } if (FORMAT_XML.equals(format)) { response.setContentType("text/xml; charset=utf-8"); } else if (FORMAT_JSON.equals(format)) { response.setContentType("application/json; charset=utf-8"); } Configuration conf = (Configuration) getServletContext().getAttribute(CONF_CONTEXT_ATTRIBUTE); assert conf != null; Writer out = response.getWriter(); if (FORMAT_JSON.equals(format)) { Configuration.dumpConfiguration(conf, out); } else if (FORMAT_XML.equals(format)) { conf.writeXml(out); } else { response.sendError(HttpServletResponse.SC_BAD_REQUEST, "Bad format: " + escapeHtml(format)); } out.close(); }
From source file:org.apache.slider.client.SliderClient.java
License:Apache License
private void actionDiagnosticYarn(ActionDiagnosticArgs diagnosticArgs) throws IOException, YarnException { JSONObject converter = null;/*from ww w . j a v a 2 s.com*/ log.info("the node in the YARN cluster has below state: "); List<NodeReport> yarnClusterInfo; try { yarnClusterInfo = yarnClient.getNodeReports(NodeState.RUNNING); } catch (YarnException e1) { log.error("Exception happened when fetching node report from the YARN cluster: " + e1.toString()); throw e1; } catch (IOException e1) { log.error("Network problem happened when fetching node report YARN cluster: " + e1.toString()); throw e1; } for (NodeReport nodeReport : yarnClusterInfo) { log.info(nodeReport.toString()); } if (diagnosticArgs.verbose) { Writer configWriter = new StringWriter(); try { Configuration.dumpConfiguration(yarnClient.getConfig(), configWriter); } catch (IOException e1) { log.error("Network problem happened when retrieving YARN config from YARN: " + e1.toString()); throw e1; } try { converter = new JSONObject(configWriter.toString()); log.info("the configuration of the YARN cluster is: " + converter.toString(2)); } catch (JSONException e) { log.error("JSONException happened during parsing response from YARN: " + e.toString()); } } }
From source file:org.godhuli.rhipe.Rbase.Admin.java
License:Apache License
public void dumpConfiguration() throws IOException { java.io.OutputStreamWriter writer = new java.io.OutputStreamWriter(System.out); Configuration.dumpConfiguration(_cfg, writer); }
From source file:org.janusgraph.hadoop.compat.h2.ImmutableConfiguration.java
License:Apache License
public static void dumpConfiguration(Configuration config, Writer out) throws IOException { Configuration.dumpConfiguration(config, out); }