Example usage for org.apache.hadoop.conf Configuration addResource

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration addResource.

Prototype

public void addResource(Configuration conf) 

Source Link

Document

Add a configuration resource.

Usage

From source file:com.flipkart.foxtrot.core.datastore.impl.hbase.HBaseUtil.java

License:Apache License

public static Configuration create(final HbaseConfig hbaseConfig) throws IOException {
    Configuration configuration = HBaseConfiguration.create();

    if (isValidFile(hbaseConfig.getCoreSite())) {
        configuration.addResource(new File(hbaseConfig.getCoreSite()).toURI().toURL());
    }/*from ww w. j  a v a 2s .c  o  m*/

    if (isValidFile(hbaseConfig.getHdfsSite())) {
        configuration.addResource(new File(hbaseConfig.getHdfsSite()).toURI().toURL());
    }

    if (isValidFile(hbaseConfig.getHbasePolicy())) {
        configuration.addResource(new File(hbaseConfig.getHbasePolicy()).toURI().toURL());
    }

    if (isValidFile(hbaseConfig.getHbaseSite())) {
        configuration.addResource(new File(hbaseConfig.getHbaseSite()).toURI().toURL());
    }

    if (hbaseConfig.isSecure() && isValidFile(hbaseConfig.getKeytabFileName())) {
        configuration.set("hbase.master.kerberos.principal", hbaseConfig.getAuthString());
        configuration.set("hadoop.kerberos.kinit.command", hbaseConfig.getKinitPath());
        UserGroupInformation.setConfiguration(configuration);
        System.setProperty("java.security.krb5.conf", hbaseConfig.getKerberosConfigFile());
        UserGroupInformation.loginUserFromKeytab(hbaseConfig.getAuthString(), hbaseConfig.getKeytabFileName());
        logger.info("Logged into Hbase with User: " + UserGroupInformation.getLoginUser());
    }

    if (null != hbaseConfig.getHbaseZookeeperQuorum()) {
        configuration.set("hbase.zookeeper.quorum", hbaseConfig.getHbaseZookeeperQuorum());
    }

    if (null != hbaseConfig.getHbaseZookeeperClientPort()) {
        configuration.setInt("hbase.zookeeper.property.clientPort", hbaseConfig.getHbaseZookeeperClientPort());
    }
    return configuration;
}

---

From source file:com.gemstone.gemfire.cache.hdfs.internal.HDFSStoreImpl.java

License:Apache License

private FileSystem createFileSystem(Configuration hconf, String configFile, boolean forceNew)
        throws IOException {
    FileSystem filesystem = null;

    // load hdfs client config file if specified. The path is on local file
    // system// w w  w .  j a va 2  s  . c  o  m
    if (configFile != null) {
        if (logger.isDebugEnabled()) {
            logger.debug("{}Adding resource config file to hdfs configuration:" + configFile, logPrefix);
        }
        hconf.addResource(new Path(configFile));

        if (!new File(configFile).exists()) {
            logger.warn(LocalizedMessage.create(LocalizedStrings.HOPLOG_HDFS_CLIENT_CONFIG_FILE_ABSENT,
                    configFile));
        }
    }

    // This setting disables shutdown hook for file system object. Shutdown
    // hook may cause FS object to close before the cache or store and
    // unpredictable behavior. This setting is provided for GFXD like server
    // use cases where FS close is managed by a server. This setting is not
    // supported by old versions of hadoop, HADOOP-4829
    hconf.setBoolean("fs.automatic.close", false);

    // Hadoop has a configuration parameter io.serializations that is a list of serialization 
    // classes which can be used for obtaining serializers and deserializers. This parameter 
    // by default contains avro classes. When a sequence file is created, it calls 
    // SerializationFactory.getSerializer(keyclass). This internally creates objects using 
    // reflection of all the classes that were part of io.serializations. But since, there is 
    // no avro class available it throws an exception. 
    // Before creating a sequenceFile, override the io.serializations parameter and pass only the classes 
    // that are important to us. 
    hconf.setStrings("io.serializations",
            new String[] { "org.apache.hadoop.io.serializer.WritableSerialization" });
    // create writer

    SchemaMetrics.configureGlobally(hconf);

    String nameNodeURL = null;
    if ((nameNodeURL = getNameNodeURL()) == null) {
        nameNodeURL = hconf.get("fs.default.name");
    }

    URI namenodeURI = URI.create(nameNodeURL);

    //if (! GemFireCacheImpl.getExisting().isHadoopGfxdLonerMode()) {
    String authType = hconf.get("hadoop.security.authentication");

    //The following code handles Gemfire XD with secure HDFS
    //A static set is used to cache all known secure HDFS NameNode urls.
    UserGroupInformation.setConfiguration(hconf);

    //Compare authentication method ignoring case to make GFXD future version complaint
    //At least version 2.0.2 starts complaining if the string "kerberos" is not in all small case.
    //However it seems current version of hadoop accept the authType in any case
    if (authType.equalsIgnoreCase("kerberos")) {

        String principal = hconf.get(HoplogConfig.KERBEROS_PRINCIPAL);
        String keyTab = hconf.get(HoplogConfig.KERBEROS_KEYTAB_FILE);

        if (!PERFORM_SECURE_HDFS_CHECK) {
            if (logger.isDebugEnabled())
                logger.debug("{}Ignore secure hdfs check", logPrefix);
        } else {
            if (!secureNameNodes.contains(nameNodeURL)) {
                if (logger.isDebugEnabled())
                    logger.debug("{}Executing secure hdfs check", logPrefix);
                try {
                    filesystem = FileSystem.newInstance(namenodeURI, hconf);
                    //Make sure no IOExceptions are generated when accessing insecure HDFS. 
                    filesystem.listFiles(new Path("/"), false);
                    throw new HDFSIOException(
                            "Gemfire XD HDFS client and HDFS cluster security levels do not match. The configured HDFS Namenode is not secured.");
                } catch (IOException ex) {
                    secureNameNodes.add(nameNodeURL);
                } finally {
                    //Close filesystem to avoid resource leak
                    if (filesystem != null) {
                        closeFileSystemIgnoreError(filesystem);
                    }
                }
            }
        }

        // check to ensure the namenode principal is defined
        String nameNodePrincipal = hconf.get("dfs.namenode.kerberos.principal");
        if (nameNodePrincipal == null) {
            throw new IOException(LocalizedStrings.GF_KERBEROS_NAMENODE_PRINCIPAL_UNDEF.toLocalizedString());
        }

        // ok, the user specified a gfxd principal so we will try to login
        if (principal != null) {
            //If NameNode principal is the same as Gemfire XD principal, there is a 
            //potential security hole
            String regex = "[/@]";
            if (nameNodePrincipal != null) {
                String HDFSUser = nameNodePrincipal.split(regex)[0];
                String GFXDUser = principal.split(regex)[0];
                if (HDFSUser.equals(GFXDUser)) {
                    logger.warn(
                            LocalizedMessage.create(LocalizedStrings.HDFS_USER_IS_SAME_AS_GF_USER, GFXDUser));
                }
            }

            // a keytab must exist if the user specifies a principal
            if (keyTab == null) {
                throw new IOException(LocalizedStrings.GF_KERBEROS_KEYTAB_UNDEF.toLocalizedString());
            }

            // the keytab must exist as well
            File f = new File(keyTab);
            if (!f.exists()) {
                throw new FileNotFoundException(
                        LocalizedStrings.GF_KERBEROS_KEYTAB_FILE_ABSENT.toLocalizedString(f.getAbsolutePath()));
            }

            //Authenticate Gemfire XD principal to Kerberos KDC using Gemfire XD keytab file
            String principalWithValidHost = SecurityUtil.getServerPrincipal(principal, "");
            UserGroupInformation.loginUserFromKeytab(principalWithValidHost, keyTab);
        } else {
            logger.warn(LocalizedMessage.create(LocalizedStrings.GF_KERBEROS_PRINCIPAL_UNDEF));
        }
    }
    //}

    filesystem = getFileSystemFactory().create(namenodeURI, hconf, forceNew);

    if (logger.isDebugEnabled()) {
        logger.debug("{}Initialized FileSystem linked to " + filesystem.getUri() + " " + filesystem.hashCode(),
                logPrefix);
    }
    return filesystem;
}

---

From source file:com.github.joshelser.hbase.TombstoneTest.java

License:Apache License

void doRun() throws Exception {
    final Configuration conf = new Configuration();
    conf.addResource(new Path("/usr/local/lib/hbase/conf/hbase-site.xml"));
    final Connection conn = ConnectionFactory.createConnection(conf);
    final TableName tableName = TableName.valueOf("tombstones");
    final Admin admin = conn.getAdmin();

    log.info("Creating table");
    createTable(admin, tableName);//from   ww  w. j a  va2  s.  co  m

    final Stopwatch sw = new Stopwatch();
    final BufferedMutator bm = conn.getBufferedMutator(tableName);
    final long rowsToWrite = 1000 * 1000 * 10;
    try {
        log.info("Writing data");
        sw.start();
        writeData(bm, rowsToWrite);
        sw.stop();
        log.info("Wrote {} rows in {}ms", rowsToWrite, sw.elapsedMillis());
    } finally {
        if (null != bm) {
            bm.close();
        }
    }

    final Table table = conn.getTable(tableName);
    final long recordsToRead = 10;
    final long recordsToDelete = 1000 * 1000;
    try {
        sw.reset();
        sw.start();
        readData(table, recordsToRead);
        sw.stop();

        log.info("Took {}ms to read {} records", sw.elapsedMillis(), recordsToRead);

        for (int i = 0; i < 3; i++) {
            sw.reset();
            sw.start();
            readAndDeleteData(table, recordsToDelete);
            sw.stop();
            log.info("Took {}ms to read and delete {} records", sw.elapsedMillis(), recordsToDelete);

            sw.reset();
            sw.start();
            readData(table, recordsToRead);
            sw.stop();

            log.info("Took {}ms to read {} records", sw.elapsedMillis(), recordsToRead);
        }
    } finally {
        if (null != table) {
            table.close();
        }
    }
}

---

From source file:com.github.sakserv.minicluster.impl.KdcLocalClusterHBaseIntegrationTest.java

License:Apache License

@BeforeClass
public static void setUp() throws Exception {

    //System.setProperty("sun.security.krb5.debug", "true");

    // Force clean
    FileUtils.deleteFolder(propertyParser.getProperty(ConfigVars.ZOOKEEPER_TEMP_DIR_KEY));
    FileUtils.deleteFolder(propertyParser.getProperty(ConfigVars.HBASE_ROOT_DIR_KEY));
    FileUtils.deleteFolder(propertyParser.getProperty(ConfigVars.KDC_BASEDIR_KEY));

    // KDC/*from   w  ww .j  a v a  2 s.co  m*/
    kdcLocalCluster = new KdcLocalCluster.Builder()
            .setPort(Integer.parseInt(propertyParser.getProperty(ConfigVars.KDC_PORT_KEY)))
            .setHost(propertyParser.getProperty(ConfigVars.KDC_HOST_KEY))
            .setBaseDir(propertyParser.getProperty(ConfigVars.KDC_BASEDIR_KEY))
            .setOrgDomain(propertyParser.getProperty(ConfigVars.KDC_ORG_DOMAIN_KEY))
            .setOrgName(propertyParser.getProperty(ConfigVars.KDC_ORG_NAME_KEY))
            .setPrincipals(propertyParser.getProperty(ConfigVars.KDC_PRINCIPALS_KEY).split(","))
            .setKrbInstance(propertyParser.getProperty(ConfigVars.KDC_KRBINSTANCE_KEY))
            .setInstance(propertyParser.getProperty(ConfigVars.KDC_INSTANCE_KEY))
            .setTransport(propertyParser.getProperty(ConfigVars.KDC_TRANSPORT))
            .setMaxTicketLifetime(
                    Integer.parseInt(propertyParser.getProperty(ConfigVars.KDC_MAX_TICKET_LIFETIME_KEY)))
            .setMaxRenewableLifetime(
                    Integer.parseInt(propertyParser.getProperty(ConfigVars.KDC_MAX_RENEWABLE_LIFETIME)))
            .setDebug(Boolean.parseBoolean(propertyParser.getProperty(ConfigVars.KDC_DEBUG))).build();
    kdcLocalCluster.start();

    Configuration baseConf = kdcLocalCluster.getBaseConf();

    // Zookeeper
    Jaas jaas = new Jaas().addServiceEntry("Server", kdcLocalCluster.getKrbPrincipal("zookeeper"),
            kdcLocalCluster.getKeytabForPrincipal("zookeeper"), "zookeeper");
    javax.security.auth.login.Configuration.setConfiguration(jaas);

    Map<String, Object> properties = new HashMap<>();
    properties.put("authProvider.1", "org.apache.zookeeper.server.auth.SASLAuthenticationProvider");
    properties.put("requireClientAuthScheme", "sasl");
    properties.put("sasl.serverconfig", "Server");
    properties.put("kerberos.removeHostFromPrincipal", "true");
    properties.put("kerberos.removeRealmFromPrincipal", "true");

    zookeeperLocalCluster = new ZookeeperLocalCluster.Builder()
            .setPort(Integer.parseInt(propertyParser.getProperty(ConfigVars.ZOOKEEPER_PORT_KEY)))
            .setTempDir(propertyParser.getProperty(ConfigVars.ZOOKEEPER_TEMP_DIR_KEY))
            .setZookeeperConnectionString(
                    propertyParser.getProperty(ConfigVars.ZOOKEEPER_CONNECTION_STRING_KEY))
            .setCustomProperties(properties).build();
    zookeeperLocalCluster.start();

    // HBase
    UserGroupInformation.setConfiguration(baseConf);

    System.setProperty("zookeeper.sasl.client", "true");
    System.setProperty("zookeeper.sasl.clientconfig", "Client");
    javax.security.auth.login.Configuration.setConfiguration(new Jaas().addEntry("Client",
            kdcLocalCluster.getKrbPrincipalWithRealm("hbase"), kdcLocalCluster.getKeytabForPrincipal("hbase")));

    try (CuratorFramework client = CuratorFrameworkFactory.newClient(
            zookeeperLocalCluster.getZookeeperConnectionString(), new ExponentialBackoffRetry(1000, 3))) {
        client.start();

        List<ACL> perms = new ArrayList<>();
        perms.add(new ACL(ZooDefs.Perms.ALL, ZooDefs.Ids.AUTH_IDS));
        perms.add(new ACL(ZooDefs.Perms.READ, ZooDefs.Ids.ANYONE_ID_UNSAFE));

        client.create().withMode(CreateMode.PERSISTENT).withACL(perms)
                .forPath(propertyParser.getProperty(ConfigVars.HBASE_ZNODE_PARENT_KEY));
    }

    Jaas jaasHbaseClient = new Jaas().addEntry("Client", kdcLocalCluster.getKrbPrincipalWithRealm("hbase"),
            kdcLocalCluster.getKeytabForPrincipal("hbase"));
    javax.security.auth.login.Configuration.setConfiguration(jaasHbaseClient);
    File jaasHbaseClientFile = new File(propertyParser.getProperty(ConfigVars.KDC_BASEDIR_KEY),
            "hbase-client.jaas");
    org.apache.commons.io.FileUtils.writeStringToFile(jaasHbaseClientFile, jaasHbaseClient.toFile());

    Configuration hbaseConfig = HBaseConfiguration.create();
    hbaseConfig.addResource(baseConf);

    hbaseLocalCluster = new HbaseLocalCluster.Builder()
            .setHbaseMasterPort(Integer.parseInt(propertyParser.getProperty(ConfigVars.HBASE_MASTER_PORT_KEY)))
            .setHbaseMasterInfoPort(
                    Integer.parseInt(propertyParser.getProperty(ConfigVars.HBASE_MASTER_INFO_PORT_KEY)))
            .setNumRegionServers(
                    Integer.parseInt(propertyParser.getProperty(ConfigVars.HBASE_NUM_REGION_SERVERS_KEY)))
            .setHbaseRootDir(propertyParser.getProperty(ConfigVars.HBASE_ROOT_DIR_KEY))
            .setZookeeperPort(Integer.parseInt(propertyParser.getProperty(ConfigVars.ZOOKEEPER_PORT_KEY)))
            .setZookeeperConnectionString(
                    propertyParser.getProperty(ConfigVars.ZOOKEEPER_CONNECTION_STRING_KEY))
            .setZookeeperZnodeParent(propertyParser.getProperty(ConfigVars.HBASE_ZNODE_PARENT_KEY))
            .setHbaseWalReplicationEnabled(Boolean
                    .parseBoolean(propertyParser.getProperty(ConfigVars.HBASE_WAL_REPLICATION_ENABLED_KEY)))
            .setHbaseConfiguration(hbaseConfig).build();
    hbaseLocalCluster.start();
}

---

From source file:com.github.sakserv.minicluster.impl.KdcLocalClusterHdfsIntegrationTest.java

License:Apache License

@BeforeClass
public static void setUp() throws Exception {

    //System.setProperty("sun.security.krb5.debug", "true");

    // KDC//from   ww  w.  j a va 2 s .  co  m
    kdcLocalCluster = new KdcLocalCluster.Builder()
            .setPort(Integer.parseInt(propertyParser.getProperty(ConfigVars.KDC_PORT_KEY)))
            .setHost(propertyParser.getProperty(ConfigVars.KDC_HOST_KEY))
            .setBaseDir(propertyParser.getProperty(ConfigVars.KDC_BASEDIR_KEY))
            .setOrgDomain(propertyParser.getProperty(ConfigVars.KDC_ORG_DOMAIN_KEY))
            .setOrgName(propertyParser.getProperty(ConfigVars.KDC_ORG_NAME_KEY))
            .setPrincipals(propertyParser.getProperty(ConfigVars.KDC_PRINCIPALS_KEY).split(","))
            .setKrbInstance(propertyParser.getProperty(ConfigVars.KDC_KRBINSTANCE_KEY))
            .setInstance(propertyParser.getProperty(ConfigVars.KDC_INSTANCE_KEY))
            .setTransport(propertyParser.getProperty(ConfigVars.KDC_TRANSPORT))
            .setMaxTicketLifetime(
                    Integer.parseInt(propertyParser.getProperty(ConfigVars.KDC_MAX_TICKET_LIFETIME_KEY)))
            .setMaxRenewableLifetime(
                    Integer.parseInt(propertyParser.getProperty(ConfigVars.KDC_MAX_RENEWABLE_LIFETIME)))
            .setDebug(Boolean.parseBoolean(propertyParser.getProperty(ConfigVars.KDC_DEBUG))).build();
    kdcLocalCluster.start();

    Configuration baseConf = kdcLocalCluster.getBaseConf();

    //HDFS
    Configuration hdfsConfig = new HdfsConfiguration();
    hdfsConfig.addResource(baseConf);
    hdfsLocalCluster = new HdfsLocalCluster.Builder()
            .setHdfsNamenodePort(
                    Integer.parseInt(propertyParser.getProperty(ConfigVars.HDFS_NAMENODE_PORT_KEY)))
            .setHdfsNamenodeHttpPort(
                    Integer.parseInt(propertyParser.getProperty(ConfigVars.HDFS_NAMENODE_HTTP_PORT_KEY)))
            .setHdfsTempDir(propertyParser.getProperty(ConfigVars.HDFS_TEMP_DIR_KEY))
            .setHdfsNumDatanodes(
                    Integer.parseInt(propertyParser.getProperty(ConfigVars.HDFS_NUM_DATANODES_KEY)))
            .setHdfsEnablePermissions(
                    Boolean.parseBoolean(propertyParser.getProperty(ConfigVars.HDFS_ENABLE_PERMISSIONS_KEY)))
            .setHdfsFormat(Boolean.parseBoolean(propertyParser.getProperty(ConfigVars.HDFS_FORMAT_KEY)))
            .setHdfsEnableRunningUserAsProxyUser(Boolean.parseBoolean(
                    propertyParser.getProperty(ConfigVars.HDFS_ENABLE_RUNNING_USER_AS_PROXY_USER)))
            .setHdfsConfig(hdfsConfig).build();
    hdfsLocalCluster.start();
}

---

From source file:com.google.appengine.tools.mapreduce.ConfigurationXmlUtil.java

License:Apache License

/**
 * Reconstitutes a Configuration from an XML string.
 *
 * @param serializedConf an XML document in Hadoop configuration format
 * @return the Configuration corresponding to the XML
 *//*from  w  ww .ja va  2s  .co m*/
public static Configuration getConfigurationFromXml(String serializedConf) {
    Preconditions.checkNotNull(serializedConf);

    try {
        byte[] serializedConfBytes = serializedConf.getBytes("UTF8");
        ByteArrayInputStream serializedConfStream = new ByteArrayInputStream(serializedConfBytes);

        // false makes Configuration not try to read defaults from the filesystem.
        Configuration conf = new Configuration(false);
        conf.addResource(serializedConfStream);
        return conf;
    } catch (UnsupportedEncodingException e) {
        throw new RuntimeException("JDK doesn't understand UTF8", e);
    }
}

---

From source file:com.google.cloud.bigtable.hbase.IntegrationTests.java

License:Open Source License

protected static void addExtraResources(Configuration configuration) {
    String extraResources = System.getProperty("bigtable.test.extra.resources");
    if (extraResources != null) {
        InputStream resourceStream = AbstractTest.class.getClassLoader().getResourceAsStream(extraResources);
        if (resourceStream != null) {
            configuration.addResource(resourceStream);
        } else {/*w ww .  java2 s  . c  o m*/
            Assert.fail(extraResources + " was not found");
        }
    }
}

---

From source file:com.iflytek.spider.util.SpiderConfiguration.java

License:Apache License

/**
 * Add the standard Spider resources to {@link Configuration}.
 * //from   ww  w.  ja va2s . co m
 * @param conf
 *            Configuration object to which configuration is to be added.
 * @param crawlConfiguration
 *            Whether configuration for command line crawl using 'bin/Spider
 *            crawl' command should be added.
 */
private static Configuration addSpiderResources(Configuration conf) {
    conf.addResource("spider.xml");
    return conf;
}

---

From source file:com.ikanow.aleph2.analytics.hadoop.utils.HadoopTechnologyUtils.java

License:Apache License

/** Generates a Hadoop configuration object
 * @param globals - the global configuration bean (containig the location of the files)
 * @return//from w  w w. j a va 2s  .  co  m
 */
public static Configuration getHadoopConfig(final GlobalPropertiesBean globals) {
    final Configuration configuration = new Configuration(false);

    if (new File(globals.local_yarn_config_dir()).exists()) {
        configuration.addResource(new Path(globals.local_yarn_config_dir() + "/core-site.xml"));
        configuration.addResource(new Path(globals.local_yarn_config_dir() + "/yarn-site.xml"));
        configuration.addResource(new Path(globals.local_yarn_config_dir() + "/hdfs-site.xml"));
        configuration.addResource(new Path(globals.local_yarn_config_dir() + "/hadoop-site.xml"));
        configuration.addResource(new Path(globals.local_yarn_config_dir() + "/mapred-site.xml"));
    }
    // These are not added by Hortonworks, so add them manually
    configuration.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
    configuration.set("fs.AbstractFileSystem.hdfs.impl", "org.apache.hadoop.fs.Hdfs");
    // Some other config defaults:
    // (not sure if these are actually applied, or derived from the defaults - for some reason they don't appear in CDH's client config)
    configuration.set("mapred.reduce.tasks.speculative.execution", "false");

    return configuration;
}

---

From source file:com.ikanow.aleph2.analytics.r.utils.HadoopTechnologyUtils.java

License:Apache License

/** Generates a Hadoop configuration object
 * @param attempt - handles retries necessary because of the concurrent issues
 * @param globals - the global configuration bean (containig the location of the files)
 * @return//w w w .  j a va  2 s  .c  o m
 */
public static Configuration getHadoopConfig(long attempt, final GlobalPropertiesBean globals) {
    synchronized (Configuration.class) { // (this is not thread safe)
        final Configuration configuration = new Configuration(false);

        if (new File(globals.local_yarn_config_dir()).exists()) {
            configuration.addResource(new Path(globals.local_yarn_config_dir() + "/core-site.xml"));
            configuration.addResource(new Path(globals.local_yarn_config_dir() + "/yarn-site.xml"));
            configuration.addResource(new Path(globals.local_yarn_config_dir() + "/hdfs-site.xml"));
            configuration.addResource(new Path(globals.local_yarn_config_dir() + "/hadoop-site.xml"));
            configuration.addResource(new Path(globals.local_yarn_config_dir() + "/mapred-site.xml"));
        }
        if (attempt > 10) { // (try sleeping here)
            final long to_sleep = 500L + (new Date().getTime() % 100L); // (add random component)
            try {
                Thread.sleep(to_sleep);
            } catch (Exception e) {
            }
        }
        // These are not added by Hortonworks, so add them manually
        configuration.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
        configuration.set("fs.file.impl", "org.apache.hadoop.fs.LocalFileSystem");
        configuration.set("fs.AbstractFileSystem.hdfs.impl", "org.apache.hadoop.fs.Hdfs");
        configuration.set("fs.AbstractFileSystem.file.impl", "org.apache.hadoop.fs.local.LocalFs");
        // Some other config defaults:
        // (not sure if these are actually applied, or derived from the defaults - for some reason they don't appear in CDH's client config)
        configuration.set("mapred.reduce.tasks.speculative.execution", "false");

        return configuration;
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »