List of usage examples for org.apache.hadoop.conf Configuration writeXml
public void writeXml(Writer out) throws IOException
From source file:org.apache.oozie.test.XTestCase.java
License:Apache License
/** * Like {@link #setUp()} but allows skipping cleaning up the database tables. Most tests should use the other method, unless * they specifically don't want to (or can't) clean up the database tables. * * @param cleanUpDBTables true if should cleanup the database tables, false if not * @throws Exception if the test workflow working directory could not be created or there was a problem cleaning the database *//*from www. j av a 2s . c om*/ protected void setUp(boolean cleanUpDBTables) throws Exception { RUNNING_TESTCASES.incrementAndGet(); super.setUp(); String baseDir = System.getProperty(OOZIE_TEST_DIR, new File("target/test-data").getAbsolutePath()); String msg = null; File f = new File(baseDir); if (!f.isAbsolute()) { msg = XLog.format("System property [{0}]=[{1}] must be set to an absolute path", OOZIE_TEST_DIR, baseDir); } else { if (baseDir.length() < 4) { msg = XLog.format("System property [{0}]=[{1}] path must be at least 4 chars", OOZIE_TEST_DIR, baseDir); } } if (msg != null) { System.err.println(); System.err.println(msg); System.exit(-1); } f.mkdirs(); if (!f.exists() || !f.isDirectory()) { System.err.println(); System.err.println(XLog.format("Could not create test dir [{0}]", baseDir)); System.exit(-1); } hadoopVersion = System.getProperty(HADOOP_VERSION, "0.20.0"); testCaseDir = createTestCaseDir(this, true); //setting up Oozie HOME and Oozie conf directory setSystemProperty(Services.OOZIE_HOME_DIR, testCaseDir); Services.setOozieHome(); testCaseConfDir = createTestCaseSubDir("conf"); // load test Oozie site String oozieTestDB = System.getProperty("oozie.test.db", "hsqldb"); String defaultOozieSize = new File(OOZIE_SRC_DIR, "core/src/test/resources/" + oozieTestDB + "-oozie-site.xml").getAbsolutePath(); String customOozieSite = System.getProperty("oozie.test.config.file", defaultOozieSize); File source = new File(customOozieSite); if (!source.isAbsolute()) { source = new File(OOZIE_SRC_DIR, customOozieSite); } source = source.getAbsoluteFile(); InputStream oozieSiteSourceStream = null; if (source.exists()) { oozieSiteSourceStream = new FileInputStream(source); } else { // If we can't find it, try using the class loader (useful if we're using XTestCase from outside core) URL sourceURL = getClass().getClassLoader().getResource(oozieTestDB + "-oozie-site.xml"); if (sourceURL != null) { oozieSiteSourceStream = sourceURL.openStream(); } else { // If we still can't find it, then exit System.err.println(); System.err.println(XLog.format("Custom configuration file for testing does no exist [{0}]", source.getAbsolutePath())); System.err.println(); System.exit(-1); } } // Copy the specified oozie-site file from oozieSiteSourceStream to the test case dir as oozie-site.xml Configuration oozieSiteConf = new Configuration(false); oozieSiteConf.addResource(oozieSiteSourceStream); ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); InputStream inputStream = classLoader.getResourceAsStream(ConfigurationService.DEFAULT_CONFIG_FILE); XConfiguration configuration = new XConfiguration(inputStream); String classes = configuration.get(Services.CONF_SERVICE_CLASSES); // Disable sharelib service as it cannot find the sharelib jars // as maven has target/classes in classpath and not the jar because test phase is before package phase oozieSiteConf.set(Services.CONF_SERVICE_CLASSES, classes.replaceAll("org.apache.oozie.service.ShareLibService,", "")); // Make sure to create the Oozie DB during unit tests oozieSiteConf.set(JPAService.CONF_CREATE_DB_SCHEMA, "true"); File target = new File(testCaseConfDir, "oozie-site.xml"); oozieSiteConf.writeXml(new FileOutputStream(target)); File hadoopConfDir = new File(testCaseConfDir, "hadoop-conf"); hadoopConfDir.mkdir(); File actionConfDir = new File(testCaseConfDir, "action-conf"); actionConfDir.mkdir(); source = new File(OOZIE_SRC_DIR, "core/src/test/resources/hadoop-config.xml"); target = new File(hadoopConfDir, "hadoop-site.xml"); IOUtils.copyStream(new FileInputStream(source), new FileOutputStream(target)); if (System.getProperty("oozielocal.log") == null) { setSystemProperty("oozielocal.log", "/tmp/oozielocal.log"); } if (System.getProperty("oozie.test.hadoop.security", "simple").equals("kerberos")) { System.setProperty("oozie.service.HadoopAccessorService.kerberos.enabled", "true"); } if (System.getProperty("oozie.test.hadoop.minicluster", "true").equals("true")) { setUpEmbeddedHadoop(getTestCaseDir()); // Second cluster is not necessary without the first one if (System.getProperty("oozie.test.hadoop.minicluster2", "false").equals("true")) { setUpEmbeddedHadoop2(); } } if (System.getProperty("oozie.test.db.host") == null) { System.setProperty("oozie.test.db.host", "localhost"); } setSystemProperty(ConfigurationService.OOZIE_DATA_DIR, testCaseDir); setSystemProperty(HadoopAccessorService.SUPPORTED_FILESYSTEMS, "*"); if (mrCluster != null) { OutputStream os = new FileOutputStream(new File(hadoopConfDir, "core-site.xml")); Configuration conf = createJobConfFromMRCluster(); conf.writeXml(os); os.close(); } if (System.getProperty("oozie.test.metastore.server", "true").equals("true")) { setupHCatalogServer(); } // Cleanup any leftover database data to make sure we start each test with an empty database if (cleanUpDBTables) { cleanUpDBTables(); } }
From source file:org.apache.oozie.tools.diag.ServerInfoCollector.java
License:Apache License
void storeServerConfiguration(File outputDir) { try {/*from w w w . j a va2s .com*/ System.out.print("Getting Configuration..."); final Map<String, String> serverConfigMap = client.getServerConfiguration(); final Configuration serverConfig = new Configuration(false); for (Map.Entry<String, String> ent : serverConfigMap.entrySet()) { serverConfig.set(ent.getKey(), ent.getValue()); } try (OutputStream outputStream = new FileOutputStream( new File(outputDir, "effective-oozie-site.xml"))) { serverConfig.writeXml(outputStream); } System.out.println("Done"); } catch (OozieClientException | IOException e) { System.err.printf("Exception occurred during the retrieval of effective Oozie server configuration " + "\"oozie-site.xml\": %s%n", e.getMessage()); } }
From source file:org.apache.phoenix.mapreduce.CsvBulkImportUtilTest.java
License:Apache License
@Test public void testInitCsvImportJob() throws IOException { Configuration conf = new Configuration(); char delimiter = '\001'; char quote = '\002'; char escape = '!'; CsvBulkImportUtil.initCsvImportJob(conf, delimiter, quote, escape, null, null); // Serialize and deserialize the config to ensure that there aren't any issues // with non-printable characters as delimiters File tempFile = File.createTempFile("test-config", ".xml"); FileOutputStream fileOutputStream = new FileOutputStream(tempFile); conf.writeXml(fileOutputStream); fileOutputStream.close();/*from ww w .ja va 2 s . c o m*/ Configuration deserialized = new Configuration(); deserialized.addResource(new FileInputStream(tempFile)); assertEquals(Character.valueOf('\001'), CsvBulkImportUtil.getCharacter(deserialized, CsvToKeyValueMapper.FIELD_DELIMITER_CONFKEY)); assertEquals(Character.valueOf('\002'), CsvBulkImportUtil.getCharacter(deserialized, CsvToKeyValueMapper.QUOTE_CHAR_CONFKEY)); assertEquals(Character.valueOf('!'), CsvBulkImportUtil.getCharacter(deserialized, CsvToKeyValueMapper.ESCAPE_CHAR_CONFKEY)); assertNull(deserialized.get(CsvToKeyValueMapper.ARRAY_DELIMITER_CONFKEY)); tempFile.delete(); }
From source file:org.apache.pig.test.TezMiniCluster.java
License:Apache License
@Override public void setupMiniDfsAndMrClusters() { try {//from w ww .j ava 2 s. c o m deleteConfFiles(); CONF_DIR.mkdirs(); // Build mini DFS cluster Configuration hdfsConf = new Configuration(false); hdfsConf.addResource("core-default.xml"); hdfsConf.addResource("hdfs-default.xml"); m_dfs = new MiniDFSCluster.Builder(hdfsConf).numDataNodes(2).format(true).racks(null).build(); m_fileSys = m_dfs.getFileSystem(); m_dfs_conf = m_dfs.getConfiguration(0); //Create user home directory m_fileSys.mkdirs(m_fileSys.getWorkingDirectory()); // Write core-site.xml Configuration core_site = new Configuration(false); core_site.set(FileSystem.FS_DEFAULT_NAME_KEY, m_dfs_conf.get(FileSystem.FS_DEFAULT_NAME_KEY)); core_site.writeXml(new FileOutputStream(CORE_CONF_FILE)); Configuration hdfs_site = new Configuration(false); for (Entry<String, String> conf : m_dfs_conf) { if (ArrayUtils.contains(m_dfs_conf.getPropertySources(conf.getKey()), "programatically")) { hdfs_site.set(conf.getKey(), m_dfs_conf.getRaw(conf.getKey())); } } hdfs_site.writeXml(new FileOutputStream(HDFS_CONF_FILE)); // Build mini YARN cluster m_mr = new MiniMRYarnCluster("PigMiniCluster", 2); m_mr.init(m_dfs_conf); m_mr.start(); m_mr_conf = m_mr.getConfig(); m_mr_conf.set(MRConfiguration.FRAMEWORK_NAME, "yarn-tez"); m_mr_conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, System.getProperty("java.class.path")); m_mr_conf.set(MRJobConfig.MAP_JAVA_OPTS, "-Xmx2048m"); m_mr_conf.set(MRJobConfig.REDUCE_JAVA_OPTS, "-Xmx2048m"); Configuration mapred_site = new Configuration(false); Configuration yarn_site = new Configuration(false); for (Entry<String, String> conf : m_mr_conf) { if (ArrayUtils.contains(m_mr_conf.getPropertySources(conf.getKey()), "programatically")) { if (conf.getKey().contains("yarn")) { yarn_site.set(conf.getKey(), m_mr_conf.getRaw(conf.getKey())); } else if (!conf.getKey().startsWith("dfs")) { mapred_site.set(conf.getKey(), m_mr_conf.getRaw(conf.getKey())); } } } mapred_site.writeXml(new FileOutputStream(MAPRED_CONF_FILE)); yarn_site.writeXml(new FileOutputStream(YARN_CONF_FILE)); // Write tez-site.xml Configuration tez_conf = new Configuration(false); // TODO PIG-3659 - Remove this once memory management is fixed tez_conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, "20"); tez_conf.set("tez.lib.uris", "hdfs:///tez,hdfs:///tez/lib"); // Set to a lower value so that tests don't get stuck for long because of 1 AM running at a time tez_conf.set(TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS, "20"); // Lower the max task attempts to 2 so that negative tests fail // faster. By default, tasks retry 4 times tez_conf.set(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, "2"); tez_conf.writeXml(new FileOutputStream(TEZ_CONF_FILE)); // Copy tez jars to hdfs m_fileSys.mkdirs(new Path("/tez/lib")); FileFilter fileFilter = new RegexFileFilter("tez-.+\\.jar$"); File[] tezJars = TEZ_LIB_DIR.listFiles(fileFilter); for (int i = 0; i < tezJars.length; i++) { if (tezJars[i].getName().startsWith("tez-api")) { m_fileSys.copyFromLocalFile(new Path(tezJars[i].getAbsoluteFile().toString()), new Path("/tez")); } else { m_fileSys.copyFromLocalFile(new Path(tezJars[i].getAbsoluteFile().toString()), new Path("/tez/lib")); } } m_conf = m_mr_conf; // Turn FetchOptimizer off so that we can actually test Tez m_conf.set(PigConfiguration.OPT_FETCH, System.getProperty("test.opt.fetch", "false")); System.setProperty("junit.hadoop.conf", CONF_DIR.getPath()); System.setProperty("hadoop.log.dir", "build/test/logs"); } catch (IOException e) { throw new RuntimeException(e); } }
From source file:org.apache.sentry.api.service.thrift.ConfServlet.java
License:Apache License
@Override public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { String format = request.getParameter(FORMAT_PARAM); if (format == null) { format = FORMAT_XML;/*from w ww . ja v a 2 s .c om*/ } if (FORMAT_XML.equals(format)) { response.setContentType("text/xml; charset=utf-8"); } else if (FORMAT_JSON.equals(format)) { response.setContentType("application/json; charset=utf-8"); } Configuration conf = (Configuration) getServletContext().getAttribute(CONF_CONTEXT_ATTRIBUTE); assert conf != null; Writer out = response.getWriter(); if (FORMAT_JSON.equals(format)) { Configuration.dumpConfiguration(conf, out); } else if (FORMAT_XML.equals(format)) { conf.writeXml(out); } else { response.sendError(HttpServletResponse.SC_BAD_REQUEST, "Bad format: " + escapeHtml(format)); } out.close(); }
From source file:org.apache.slider.common.tools.ConfigHelper.java
License:Apache License
/** * Convert to an XML string//from www .ja v a 2 s . com * @param conf configuration * @return conf * @throws IOException */ public static String toXml(Configuration conf) throws IOException { StringWriter writer = new StringWriter(); conf.writeXml(writer); return writer.toString(); }
From source file:org.apache.tajo.master.TaskRunnerLauncherImpl.java
License:Apache License
private void writeConf(Configuration conf, Path queryConfFile) throws IOException { // Write job file to Tajo's fs FileSystem fs = queryConfFile.getFileSystem(conf); FSDataOutputStream out = FileSystem.create(fs, queryConfFile, new FsPermission(QUERYCONF_FILE_PERMISSION)); try {//from ww w.j av a 2 s . co m conf.writeXml(out); } finally { out.close(); } }
From source file:org.apache.tajo.master.YarnContainerProxy.java
License:Apache License
private static void writeConf(Configuration conf, Path queryConfFile) throws IOException { // Write job file to Tajo's fs FileSystem fs = queryConfFile.getFileSystem(conf); FSDataOutputStream out = FileSystem.create(fs, queryConfFile, new FsPermission(QUERYCONF_FILE_PERMISSION)); try {/*www .j a v a 2s .c o m*/ conf.writeXml(out); } finally { out.close(); } }
From source file:org.apache.tajo.yarn.command.LaunchCommand.java
License:Apache License
private void setupLocalResources(ContainerLaunchContext amContainer, FileSystem fs, ApplicationId appId) throws IOException { // set local resources for the application master // local files or archives as needed // In this scenario, the jar file for the application master is part of the local resources Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); LOG.info("Copy App Master jar from local filesystem and add to local environment"); // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path String appMasterJar = findContainingJar(ApplicationMaster.class); addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.getId(), localResources, LocalResourceType.FILE);//from ww w.j av a 2 s. co m addToLocalResources(fs, libDir, libDir, appId.getId(), localResources, LocalResourceType.FILE); addToLocalResources(fs, tajoArchive, "tajo", appId.getId(), localResources, LocalResourceType.ARCHIVE); // Set the log4j properties if needed if (!log4jPropFile.isEmpty()) { addToLocalResources(fs, log4jPropFile, log4jPath, appId.getId(), localResources, LocalResourceType.FILE); } // addToLocalResources(fs, confDir, "conf", appId.getId(), // localResources, LocalResourceType.FILE); // Tajo master conf Configuration tajoMasterConf = new Configuration(false); tajoMasterConf.addResource(new Path(confDir, "tajo-site.xml")); String suffix = appName + "/" + appId.getId() + "/master-conf"; Path dst = new Path(fs.getHomeDirectory(), suffix); fs.mkdirs(dst); Path confFile = new Path(dst, "tajo-site.xml"); FSDataOutputStream fdos = fs.create(confFile); tajoMasterConf.writeXml(fdos); fdos.close(); FileStatus scFileStatus = fs.getFileStatus(dst); LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime()); localResources.put("conf", scRsrc); amContainer.setLocalResources(localResources); }
From source file:org.apache.tajo.yarn.container.WorkerContainerTask.java
License:Apache License
@Override public ContainerLaunchContext getLaunchContext(Container container) throws IOException { // create a container launch context ContainerLaunchContext launchContext = Records.newRecord(ContainerLaunchContext.class); UserGroupInformation user = UserGroupInformation.getCurrentUser(); try {/*from ww w .j a v a 2 s . c om*/ Credentials credentials = user.getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); launchContext.setTokens(securityTokens); } catch (IOException e) { LOG.warn("Getting current user info failed when trying to launch the container" + e.getMessage()); } FileSystem fs = FileSystem.get(appContext.getConfiguration()); // Set the local resources Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); String suffix = "Tajo" + "/" + appContext.getApplicationId().getId(); Path parentPath = new Path(fs.getHomeDirectory(), suffix); // tar ball Path archivePath = new Path(parentPath, System.getenv(Constants.TAJO_ARCHIVE_PATH)); FileStatus archiveFs = fs.getFileStatus(archivePath); LocalResource archiveRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(archivePath.toUri()), LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION, archiveFs.getLen(), archiveFs.getModificationTime()); localResources.put("tajo", archiveRsrc); Configuration tajoWorkerConf = new Configuration(false); tajoWorkerConf.addResource(new Path("conf", "tajo-site.xml")); tajoWorkerConf.set(Constants.TAJO_MASTER_UMBILICAL_RPC_ADDRESS, appContext.getMasterHost() + ":26001"); tajoWorkerConf.set(Constants.CATALOG_ADDRESS, appContext.getMasterHost() + ":26005"); Path dst = new Path(parentPath, container.getId() + Path.SEPARATOR + "worker-conf"); fs.mkdirs(dst); Path confFile = new Path(dst, "tajo-site.xml"); FSDataOutputStream fdos = fs.create(confFile); tajoWorkerConf.writeXml(fdos); fdos.close(); FileStatus scFileStatus = fs.getFileStatus(dst); LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime()); localResources.put("conf", scRsrc); launchContext.setLocalResources(localResources); // Set the environment setupEnv(launchContext); // Set the necessary command to execute on the allocated container Vector<CharSequence> vargs = new Vector<CharSequence>(5); // Set executable command // Set args for the shell command if any vargs.add("${" + Constants.TAJO_HOME + "}/bin/tajo"); vargs.add("--config"); vargs.add("${" + Constants.TAJO_CONF_DIR + "}"); vargs.add("worker"); // Add log redirect params // Add log redirect params vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } List<String> commands = new ArrayList<String>(); commands.add(command.toString()); launchContext.setCommands(commands); return launchContext; }