List of usage examples for org.apache.hadoop.conf Configuration writeXml
public void writeXml(Writer out) throws IOException
From source file:etl.cmd.test.XTestCase.java
License:Apache License
/** * Initialize the test working directory. <p> If it does not exist it creates it, if it already exists it deletes * all its contents. <p> The test working directory it is not deleted after the test runs. * * @throws Exception if the test workflow working directory could not be created *//*ww w.j ava 2s . c o m*/ protected void setUp() throws Exception { RUNNING_TESTCASES.incrementAndGet(); super.setUp(); String baseDir = System.getProperty(OOZIE_TEST_DIR, new File("target/test-data").getAbsolutePath()); String msg = null; File f = new File(baseDir); if (!f.isAbsolute()) { msg = XLog.format("System property [{0}]=[{1}] must be set to an absolute path", OOZIE_TEST_DIR, baseDir); } else { if (baseDir.length() < 4) { msg = XLog.format("System property [{0}]=[{1}] path must be at least 4 chars", OOZIE_TEST_DIR, baseDir); } } if (msg != null) { System.err.println(); System.err.println(msg); System.exit(-1); } f.mkdirs(); if (!f.exists() || !f.isDirectory()) { System.err.println(); System.err.println(XLog.format("Could not create test dir [{0}]", baseDir)); System.exit(-1); } hadoopVersion = System.getProperty(HADOOP_VERSION, "0.20.0"); testCaseDir = createTestCaseDir(this, true); //setting up Oozie HOME and Oozie conf directory setSystemProperty(Services.OOZIE_HOME_DIR, testCaseDir); Services.setOozieHome(); testCaseConfDir = createTestCaseSubDir("conf"); // load test Oozie site String oozieTestDB = System.getProperty("oozie.test.db", "hsqldb"); String defaultOozieSize = new File("src/test/resources/" + oozieTestDB + "-oozie-site.xml") .getAbsolutePath(); String customOozieSite = System.getProperty("oozie.test.config.file", defaultOozieSize); File source = new File(customOozieSite); source = source.getAbsoluteFile(); InputStream oozieSiteSourceStream = null; if (source.exists()) { oozieSiteSourceStream = new FileInputStream(source); } else { // If we can't find it, try using the class loader (useful if we're using XTestCase from outside core) URL sourceURL = getClass().getClassLoader().getResource(oozieTestDB + "-oozie-site.xml"); if (sourceURL != null) { oozieSiteSourceStream = sourceURL.openStream(); } else { // If we still can't find it, then exit System.err.println(); System.err.println(XLog.format("Custom configuration file for testing does no exist [{0}]", source.getAbsolutePath())); System.err.println(); System.exit(-1); } } // Copy the specified oozie-site file from oozieSiteSourceStream to the test case dir as oozie-site.xml Configuration oozieSiteConf = new Configuration(false); oozieSiteConf.addResource(oozieSiteSourceStream); ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); InputStream inputStream = classLoader.getResourceAsStream(ConfigurationService.DEFAULT_CONFIG_FILE); XConfiguration configuration = new XConfiguration(inputStream); String classes = configuration.get(Services.CONF_SERVICE_CLASSES); // Disable sharelib service as it cannot find the sharelib jars // as maven has target/classes in classpath and not the jar because test phase is before package phase // if (System.getProperty("oozie.test.hadoop.minicluster", "true").equals("true")) oozieSiteConf.set(Services.CONF_SERVICE_CLASSES, classes.replaceAll("org.apache.oozie.service.ShareLibService,", "")); // Make sure to create the Oozie DB during unit tests oozieSiteConf.set(JPAService.CONF_CREATE_DB_SCHEMA, "true"); File target = new File(testCaseConfDir, "oozie-site.xml"); oozieSiteConf.writeXml(new FileOutputStream(target)); File hadoopConfDir = new File(testCaseConfDir, "hadoop-conf"); hadoopConfDir.mkdir(); File actionConfDir = new File(testCaseConfDir, "action-conf"); actionConfDir.mkdir(); source = new File("src/test/resources/hadoop-config.xml"); target = new File(hadoopConfDir, "core-site.xml"); IOUtils.copyStream(new FileInputStream(source), new FileOutputStream(target)); if (System.getProperty("oozielocal.log") == null) { setSystemProperty("oozielocal.log", "/tmp/oozielocal.log"); } if (System.getProperty("oozie.test.hadoop.security", "simple").equals("kerberos")) { System.setProperty("oozie.service.HadoopAccessorService.kerberos.enabled", "true"); } if (System.getProperty("oozie.test.hadoop.minicluster", "true").equals("true")) { setUpEmbeddedHadoop(getTestCaseDir()); // Second cluster is not necessary without the first one if (System.getProperty("oozie.test.hadoop.minicluster2", "false").equals("true")) { setUpEmbeddedHadoop2(); } } if (System.getProperty("oozie.test.db.host") == null) { System.setProperty("oozie.test.db.host", "localhost"); } setSystemProperty(ConfigurationService.OOZIE_DATA_DIR, testCaseDir); setSystemProperty(HadoopAccessorService.SUPPORTED_FILESYSTEMS, "*"); if (mrCluster != null) { OutputStream os = new FileOutputStream(new File(hadoopConfDir, "core-site.xml")); Configuration conf = mrCluster.getConfig(); conf.writeXml(os); os.close(); } }
From source file:io.hops.security.HopsUtil.java
License:Apache License
private static void writeSSLConf(Configuration sslConf, Configuration systemConf, File passwdFile) throws IOException { String filename = systemConf.get(SSLFactory.SSL_SERVER_CONF_KEY, "ssl-server.xml"); // Workaround for testing String outputFile;/*from w w w .j a va 2 s.co m*/ if (passwdFile.getParentFile() == null) { outputFile = filename; } else { outputFile = Paths.get(passwdFile.getParentFile().getAbsolutePath(), filename).toString(); } try (FileWriter fw = new FileWriter(outputFile, false)) { sslConf.writeXml(fw); fw.flush(); } }
From source file:io.hops.tensorflow.TestCluster.java
License:Apache License
protected void setupInternal(int numNodeManager) throws Exception { LOG.info("Starting up YARN cluster"); conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); conf.set("yarn.log.dir", "target"); conf.set("yarn.log-aggregation-enable", "true"); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName()); conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); conf.setBoolean(YarnConfiguration.NM_GPU_RESOURCE_ENABLED, false); if (yarnCluster == null) { yarnCluster = new MiniYARNCluster(TestCluster.class.getSimpleName(), 1, numNodeManager, 1, 1); yarnCluster.init(conf);//from w ww . ja va 2 s .c o m yarnCluster.start(); conf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, MiniYARNCluster.getHostname() + ":" + yarnCluster.getApplicationHistoryServer().getPort()); waitForNMsToRegister(); URL url = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml"); if (url == null) { throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath"); } Configuration yarnClusterConfig = yarnCluster.getConfig(); yarnClusterConfig.set("yarn.application.classpath", new File(url.getPath()).getParent()); //write the document to a buffer (not directly to the file, as that //can cause the file being written to get read -which will then fail. ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); yarnClusterConfig.writeXml(bytesOut); bytesOut.close(); //write the bytes to the file in the classpath OutputStream os = new FileOutputStream(new File(url.getPath())); os.write(bytesOut.toByteArray()); os.close(); } FileContext fsContext = FileContext.getLocalFSFileContext(); fsContext.delete(new Path(conf.get("yarn.timeline-service.leveldb-timeline-store.path")), true); try { Thread.sleep(2000); } catch (InterruptedException e) { LOG.info("setup thread sleep interrupted. message=" + e.getMessage()); } }
From source file:ml.shifu.guagua.yarn.util.YarnUtils.java
License:Apache License
/** * Export our populated Configuration as an XML file to be used by the ApplicationMaster's exec container, and * register it with LocalResources.//from w ww . j a v a 2 s .c om * * @param conf * the current Configuration object to be published. * @param appId * the ApplicationId to stamp this app's base HDFS resources dir. */ public static void exportGuaguaConfiguration(Configuration conf, ApplicationId appId) throws IOException { FileSystem fs = FileSystem.get(conf); Path hdfsConfPath = new Path(getAppDirectory(fs, appId), GuaguaYarnConstants.GUAGUA_CONF_FILE); FSDataOutputStream fos = null; try { fos = FileSystem.get(conf).create(hdfsConfPath, true); conf.writeXml(fos); fos.flush(); } finally { if (null != fos) { fos.close(); } } }
From source file:org.apache.accumulo.harness.MiniClusterHarness.java
License:Apache License
public MiniAccumuloClusterImpl create(String testClassName, String testMethodName, AuthenticationToken token, MiniClusterConfigurationCallback configCallback, TestingKdc kdc) throws Exception { requireNonNull(token);/*from w w w. ja v a2 s .c o m*/ checkArgument(token instanceof PasswordToken || token instanceof KerberosToken, "A PasswordToken or KerberosToken is required"); String rootPasswd; if (token instanceof PasswordToken) { rootPasswd = new String(((PasswordToken) token).getPassword(), UTF_8); } else { rootPasswd = UUID.randomUUID().toString(); } File baseDir = AccumuloClusterHarness.createTestDir(testClassName + "_" + testMethodName); MiniAccumuloConfigImpl cfg = new MiniAccumuloConfigImpl(baseDir, rootPasswd); // Enable native maps by default cfg.setNativeLibPaths(NativeMapIT.nativeMapLocation().getAbsolutePath()); cfg.setProperty(Property.TSERV_NATIVEMAP_ENABLED, Boolean.TRUE.toString()); Configuration coreSite = new Configuration(false); // Setup SSL and credential providers if the properties request such configureForEnvironment(cfg, getClass(), AccumuloClusterHarness.getSslDir(baseDir), coreSite, kdc); // Invoke the callback for tests to configure MAC before it starts configCallback.configureMiniCluster(cfg, coreSite); MiniAccumuloClusterImpl miniCluster = new MiniAccumuloClusterImpl(cfg); // Write out any configuration items to a file so HDFS will pick them up automatically (from the classpath) if (coreSite.size() > 0) { File csFile = new File(miniCluster.getConfig().getConfDir(), "core-site.xml"); if (csFile.exists()) throw new RuntimeException(csFile + " already exist"); OutputStream out = new BufferedOutputStream( new FileOutputStream(new File(miniCluster.getConfig().getConfDir(), "core-site.xml"))); coreSite.writeXml(out); out.close(); } return miniCluster; }
From source file:org.apache.accumulo.server.util.Admin.java
License:Apache License
private void printSystemConfiguration(Connector connector, File outputDirectory) throws IOException, AccumuloException, AccumuloSecurityException { Configuration conf = new Configuration(false); TreeMap<String, String> site = new TreeMap<>(siteConfig); for (Entry<String, String> prop : site.entrySet()) { String defaultValue = getDefaultConfigValue(prop.getKey()); if (!prop.getValue().equals(defaultValue) && !systemConfig.containsKey(prop.getKey())) { conf.set(prop.getKey(), prop.getValue()); }//ww w . ja va 2s .c om } TreeMap<String, String> system = new TreeMap<>(systemConfig); for (Entry<String, String> prop : system.entrySet()) { String defaultValue = getDefaultConfigValue(prop.getKey()); if (!prop.getValue().equals(defaultValue)) { conf.set(prop.getKey(), prop.getValue()); } } File siteBackup = new File(outputDirectory, ACCUMULO_SITE_BACKUP_FILE); FileOutputStream fos = new FileOutputStream(siteBackup); try { conf.writeXml(fos); } finally { fos.close(); } }
From source file:org.apache.accumulo.test.ExistingMacIT.java
License:Apache License
private void createEmptyConfig(File confFile) throws IOException { Configuration conf = new Configuration(false); OutputStream hcOut = new FileOutputStream(confFile); conf.writeXml(hcOut); hcOut.close();//from w w w. j a v a 2 s . com }
From source file:org.apache.accumulo.test.functional.ConfigurableMacBase.java
License:Apache License
private void createMiniAccumulo() throws Exception { // createTestDir will give us a empty directory, we don't need to clean it up ourselves File baseDir = createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName()); MiniAccumuloConfigImpl cfg = new MiniAccumuloConfigImpl(baseDir, ROOT_PASSWORD); String nativePathInDevTree = NativeMapIT.nativeMapLocation().getAbsolutePath(); String nativePathInMapReduce = new File(System.getProperty("user.dir")).toString(); cfg.setNativeLibPaths(nativePathInDevTree, nativePathInMapReduce); cfg.setProperty(Property.GC_FILE_ARCHIVE, Boolean.TRUE.toString()); Configuration coreSite = new Configuration(false); configure(cfg, coreSite);// w ww . java 2 s.c om cfg.setProperty(Property.TSERV_NATIVEMAP_ENABLED, Boolean.TRUE.toString()); configureForEnvironment(cfg, getClass(), getSslDir(baseDir)); cluster = new MiniAccumuloClusterImpl(cfg); if (coreSite.size() > 0) { File csFile = new File(cluster.getConfig().getConfDir(), "core-site.xml"); if (csFile.exists()) { coreSite.addResource(new Path(csFile.getAbsolutePath())); } File tmp = new File(csFile.getAbsolutePath() + ".tmp"); OutputStream out = new BufferedOutputStream(new FileOutputStream(tmp)); coreSite.writeXml(out); out.close(); assertTrue(tmp.renameTo(csFile)); } beforeClusterStart(cfg); }
From source file:org.apache.accumulo.test.functional.ConfigurableMacIT.java
License:Apache License
@Before public void setUp() throws Exception { MiniAccumuloConfigImpl cfg = new MiniAccumuloConfigImpl( createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName()), ROOT_PASSWORD); cfg.setNativeLibPaths(NativeMapIT.nativeMapLocation().getAbsolutePath()); Configuration coreSite = new Configuration(false); configure(cfg, coreSite);/*from w ww. j av a 2 s. c o m*/ cfg.setProperty(Property.TSERV_NATIVEMAP_ENABLED, Boolean.TRUE.toString()); configureForEnvironment(cfg, getClass(), createSharedTestDir(this.getClass().getName() + "-ssl")); cluster = new MiniAccumuloClusterImpl(cfg); if (coreSite.size() > 0) { File csFile = new File(cluster.getConfig().getConfDir(), "core-site.xml"); if (csFile.exists()) throw new RuntimeException(csFile + " already exist"); OutputStream out = new BufferedOutputStream( new FileOutputStream(new File(cluster.getConfig().getConfDir(), "core-site.xml"))); coreSite.writeXml(out); out.close(); } beforeClusterStart(cfg); cluster.start(); }
From source file:org.apache.accumulo.test.replication.CyclicReplicationIT.java
License:Apache License
private void setCoreSite(MiniAccumuloClusterImpl cluster) throws Exception { File csFile = new File(cluster.getConfig().getConfDir(), "core-site.xml"); if (csFile.exists()) throw new RuntimeException(csFile + " already exist"); Configuration coreSite = new Configuration(false); coreSite.set("fs.file.impl", RawLocalFileSystem.class.getName()); OutputStream out = new BufferedOutputStream( new FileOutputStream(new File(cluster.getConfig().getConfDir(), "core-site.xml"))); coreSite.writeXml(out); out.close();/*from w w w . j av a2 s . com*/ }