List of usage examples for org.apache.hadoop.fs FileSystem FS_DEFAULT_NAME_KEY
String FS_DEFAULT_NAME_KEY
To view the source code for org.apache.hadoop.fs FileSystem FS_DEFAULT_NAME_KEY.
Click Source Link
From source file:org.apache.flink.yarn.YarnFileStageTest.java
License:Apache License
@Before public void initConfig() { hadoopConfig = new org.apache.hadoop.conf.Configuration(); hadoopConfig.set(org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY, hdfsRootPath.toString()); }
From source file:org.apache.kylin.common.util.HadoopUtilTest.java
License:Apache License
@Test public void testGetCurrentHBaseConfiguration() throws Exception { KylinConfig config = KylinConfig.getInstanceFromEnv(); config.setProperty(KylinConfig.KYLIN_HBASE_CLUSTER_FS, "hdfs://hbase-cluster/"); Configuration conf = HadoopUtil.getCurrentHBaseConfiguration(); assertEquals("hdfs://hbase-cluster/", conf.get(FileSystem.FS_DEFAULT_NAME_KEY)); }
From source file:org.apache.kylin.storage.hbase.HBaseConnection.java
License:Apache License
private static Configuration newHBaseConfiguration(StorageURL url) { // using a hbase:xxx URL is deprecated, instead hbase config is always loaded from hbase-site.xml in classpath if (!"hbase".equals(url.getScheme())) throw new IllegalArgumentException( "to use hbase storage, pls set 'kylin.storage.url=hbase' in kylin.properties"); Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration()); addHBaseClusterNNHAConfiguration(conf); // support hbase using a different FS KylinConfig kylinConf = KylinConfig.getInstanceFromEnv(); String hbaseClusterFs = kylinConf.getHBaseClusterFs(); if (StringUtils.isNotEmpty(hbaseClusterFs)) { conf.set(FileSystem.FS_DEFAULT_NAME_KEY, hbaseClusterFs); }//from w w w .j a v a2 s .c om // https://issues.apache.org/jira/browse/KYLIN-953 if (StringUtils.isBlank(conf.get("hadoop.tmp.dir"))) { conf.set("hadoop.tmp.dir", "/tmp"); } if (StringUtils.isBlank(conf.get("hbase.fs.tmp.dir"))) { conf.set("hbase.fs.tmp.dir", "/tmp"); } for (Entry<String, String> entry : url.getAllParameters().entrySet()) { conf.set(entry.getKey(), entry.getValue()); } return conf; }
From source file:org.apache.pig.test.TezMiniCluster.java
License:Apache License
@Override public void setupMiniDfsAndMrClusters() { try {//w w w.j a va 2s . com deleteConfFiles(); CONF_DIR.mkdirs(); // Build mini DFS cluster Configuration hdfsConf = new Configuration(false); hdfsConf.addResource("core-default.xml"); hdfsConf.addResource("hdfs-default.xml"); m_dfs = new MiniDFSCluster.Builder(hdfsConf).numDataNodes(2).format(true).racks(null).build(); m_fileSys = m_dfs.getFileSystem(); m_dfs_conf = m_dfs.getConfiguration(0); //Create user home directory m_fileSys.mkdirs(m_fileSys.getWorkingDirectory()); // Write core-site.xml Configuration core_site = new Configuration(false); core_site.set(FileSystem.FS_DEFAULT_NAME_KEY, m_dfs_conf.get(FileSystem.FS_DEFAULT_NAME_KEY)); core_site.writeXml(new FileOutputStream(CORE_CONF_FILE)); Configuration hdfs_site = new Configuration(false); for (Entry<String, String> conf : m_dfs_conf) { if (ArrayUtils.contains(m_dfs_conf.getPropertySources(conf.getKey()), "programatically")) { hdfs_site.set(conf.getKey(), m_dfs_conf.getRaw(conf.getKey())); } } hdfs_site.writeXml(new FileOutputStream(HDFS_CONF_FILE)); // Build mini YARN cluster m_mr = new MiniMRYarnCluster("PigMiniCluster", 2); m_mr.init(m_dfs_conf); m_mr.start(); m_mr_conf = m_mr.getConfig(); m_mr_conf.set(MRConfiguration.FRAMEWORK_NAME, "yarn-tez"); m_mr_conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, System.getProperty("java.class.path")); m_mr_conf.set(MRJobConfig.MAP_JAVA_OPTS, "-Xmx2048m"); m_mr_conf.set(MRJobConfig.REDUCE_JAVA_OPTS, "-Xmx2048m"); Configuration mapred_site = new Configuration(false); Configuration yarn_site = new Configuration(false); for (Entry<String, String> conf : m_mr_conf) { if (ArrayUtils.contains(m_mr_conf.getPropertySources(conf.getKey()), "programatically")) { if (conf.getKey().contains("yarn")) { yarn_site.set(conf.getKey(), m_mr_conf.getRaw(conf.getKey())); } else if (!conf.getKey().startsWith("dfs")) { mapred_site.set(conf.getKey(), m_mr_conf.getRaw(conf.getKey())); } } } mapred_site.writeXml(new FileOutputStream(MAPRED_CONF_FILE)); yarn_site.writeXml(new FileOutputStream(YARN_CONF_FILE)); // Write tez-site.xml Configuration tez_conf = new Configuration(false); // TODO PIG-3659 - Remove this once memory management is fixed tez_conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, "20"); tez_conf.set("tez.lib.uris", "hdfs:///tez,hdfs:///tez/lib"); // Set to a lower value so that tests don't get stuck for long because of 1 AM running at a time tez_conf.set(TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS, "20"); // Lower the max task attempts to 2 so that negative tests fail // faster. By default, tasks retry 4 times tez_conf.set(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, "2"); tez_conf.writeXml(new FileOutputStream(TEZ_CONF_FILE)); // Copy tez jars to hdfs m_fileSys.mkdirs(new Path("/tez/lib")); FileFilter fileFilter = new RegexFileFilter("tez-.+\\.jar$"); File[] tezJars = TEZ_LIB_DIR.listFiles(fileFilter); for (int i = 0; i < tezJars.length; i++) { if (tezJars[i].getName().startsWith("tez-api")) { m_fileSys.copyFromLocalFile(new Path(tezJars[i].getAbsoluteFile().toString()), new Path("/tez")); } else { m_fileSys.copyFromLocalFile(new Path(tezJars[i].getAbsoluteFile().toString()), new Path("/tez/lib")); } } m_conf = m_mr_conf; // Turn FetchOptimizer off so that we can actually test Tez m_conf.set(PigConfiguration.OPT_FETCH, System.getProperty("test.opt.fetch", "false")); System.setProperty("junit.hadoop.conf", CONF_DIR.getPath()); System.setProperty("hadoop.log.dir", "build/test/logs"); } catch (IOException e) { throw new RuntimeException(e); } }
From source file:org.elasticsearch.hadoop.yarn.am.ApplicationMaster.java
License:Apache License
ApplicationMaster(Map<String, String> env) {
this.env = env;
cfg = new YarnConfiguration();
if (env.containsKey(FS_URI)) {
cfg.set(FileSystem.FS_DEFAULT_NAME_KEY, env.get(FS_URI));
}/*w w w . j a v a 2 s.c o m*/
appConfig = new Config(PropertiesUtils.propsFromBase64String(env.get(CFG_PROPS)));
}
From source file:org.elasticsearch.hadoop.yarn.util.YarnUtils.java
License:Apache License
public static Map<String, String> setupEnv(Configuration cfg) { Map<String, String> env = new LinkedHashMap<String, String>(); // System.getenv() // add Hadoop Classpath for (String c : cfg.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnCompat.DEFAULT_PLATFORM_APPLICATION_CLASSPATH())) { addToEnv(env, Environment.CLASSPATH.name(), c.trim()); }//from ww w . j a v a 2s. c o m // add es-hadoop jar / current folder jars addToEnv(env, Environment.CLASSPATH.name(), "./*"); // // some es-yarn constants // addToEnv(env, EsYarnConstants.FS_URI, cfg.get(FileSystem.FS_DEFAULT_NAME_KEY, FileSystem.DEFAULT_FS)); return env; }
From source file:org.mule.modules.hdfs.automation.unit.HadoopClientConfigurationProviderTest.java
License:Open Source License
@Test public void testForSimpleAuth() { Configuration configuration = provider.forSimpleAuth(nameNodeUri, principal, configurationResources, configurationEntries);/* w w w . j a va 2 s . c o m*/ Map<String, String> expectedConfigurationEntries = new HashMap<>(); expectedConfigurationEntries.putAll(configurationEntries); expectedConfigurationEntries.put(FileSystem.FS_DEFAULT_NAME_KEY, nameNodeUri); expectedConfigurationEntries.put("hadoop.job.ugi", principal); expectedConfigurationEntries.put("hadoop.security.auth_to_local", "RULE:[1:$1@$0](hdfs-Sandbox@HDP.SANDBOX)s/.*/hdfs/"); expectedConfigurationEntries.put("dfs.namenode.kerberos.principal", "HTTP/localhost@LOCALHOST"); validateConfigurationForSimpleAuth(configuration); validateThatConfigurationContainsProperties(configuration, expectedConfigurationEntries); }
From source file:org.mule.modules.hdfs.automation.unit.HadoopClientConfigurationProviderTest.java
License:Open Source License
@Test public void testForSimpleAuthWhenPrincipalIsNull() { Configuration configuration = provider.forSimpleAuth(nameNodeUri, null, configurationResources, configurationEntries);//from ww w . j av a 2s . co m Map<String, String> expectedConfigurationEntries = new HashMap<>(); expectedConfigurationEntries.putAll(configurationEntries); expectedConfigurationEntries.put(FileSystem.FS_DEFAULT_NAME_KEY, nameNodeUri); expectedConfigurationEntries.put("hadoop.security.auth_to_local", "RULE:[1:$1@$0](hdfs-Sandbox@HDP.SANDBOX)s/.*/hdfs/"); expectedConfigurationEntries.put("dfs.namenode.kerberos.principal", "HTTP/localhost@LOCALHOST"); validateConfigurationForSimpleAuth(configuration); validateThatConfigurationContainsProperties(configuration, expectedConfigurationEntries); }
From source file:org.mule.modules.hdfs.automation.unit.HadoopClientConfigurationProviderTest.java
License:Open Source License
@Test public void testForSimpleAuthWhenPrincipalIsEmpty() { Configuration configuration = provider.forSimpleAuth(nameNodeUri, "", configurationResources, configurationEntries);// w ww .j a v a 2 s .co m Map<String, String> expectedConfigurationEntries = new HashMap<>(); expectedConfigurationEntries.putAll(configurationEntries); expectedConfigurationEntries.put(FileSystem.FS_DEFAULT_NAME_KEY, nameNodeUri); expectedConfigurationEntries.put("hadoop.security.auth_to_local", "RULE:[1:$1@$0](hdfs-Sandbox@HDP.SANDBOX)s/.*/hdfs/"); expectedConfigurationEntries.put("dfs.namenode.kerberos.principal", "HTTP/localhost@LOCALHOST"); validateConfigurationForSimpleAuth(configuration); validateThatConfigurationContainsProperties(configuration, expectedConfigurationEntries); }
From source file:org.mule.modules.hdfs.automation.unit.HadoopClientConfigurationProviderTest.java
License:Open Source License
@Test public void testForSimpleAuthWhenConfigurationResourcesIsNull() { Configuration configuration = provider.forSimpleAuth(nameNodeUri, principal, null, configurationEntries); Map<String, String> expectedConfigurationEntries = new HashMap<>(); expectedConfigurationEntries.putAll(configurationEntries); expectedConfigurationEntries.put(FileSystem.FS_DEFAULT_NAME_KEY, nameNodeUri); validateConfigurationForSimpleAuth(configuration); validateThatConfigurationContainsProperties(configuration, expectedConfigurationEntries); }