Example usage for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration.

Prototype

public YarnConfiguration(Configuration conf)

Source Link

Usage

From source file:org.elasticsearch.hadoop.yarn.am.NodeMasterRpc.java

License:Apache License

public NodeMasterRpc(Configuration cfg, NMTokenCache tokenCache) {
    this.cfg = new YarnConfiguration(cfg);
    this.tokenCache = tokenCache;
}

From source file:org.elasticsearch.hadoop.yarn.client.ClientRpc.java

License:Apache License

public ClientRpc(Configuration cfg) {
    this.cfg = new YarnConfiguration(cfg);
}

From source file:org.elasticsearch.hadoop.yarn.rpc.YarnRpc.java

License:Apache License

public YarnRpc(Class<P> protocolType, Configuration cfg) {
    this.protocolType = protocolType;
    // make a copy to avoid the security credentials spilling to the main configuration
    this.cfg = new YarnConfiguration(cfg);
}

From source file:org.huahinframework.manager.rest.service.ApplicationService.java

License:Apache License

/**
 * init/*  w ww .  j  av  a2 s .c o m*/
 */
public void init() {
    YarnRPC rpc = YarnRPC.create(getJobConf());
    YarnConfiguration yarnConf = new YarnConfiguration(getJobConf());
    InetSocketAddress rmAddress = NetUtils
            .createSocketAddr(yarnConf.get(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS));
    applicationsManager = ((ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class, rmAddress, yarnConf));
    recordFactory = RecordFactoryProvider.getRecordFactory(yarnConf);
}

From source file:org.springframework.yarn.config.annotation.builders.YarnConfigBuilder.java

License:Apache License

@Override
protected YarnConfiguration performBuild() throws Exception {
    ConfigurationFactoryBean fb = new ConfigurationFactoryBean();

    if (!loadDefaults) {
        fb.setConfiguration(new YarnConfiguration(new Configuration(false)));
    }//from  www.  java2s.c o  m

    fb.setResources(resources);
    fb.setProperties(properties);
    fb.setFsUri(fileSystemUri);
    fb.setRmAddress(rmAddress);
    fb.setSchedulerAddress(schedulerAddress);

    if (hadoopSecurity != null) {
        fb.setSecurityAuthMethod(hadoopSecurity.getSecurityAuthMethod());
        fb.setUserPrincipal(hadoopSecurity.getUserPrincipal());
        fb.setUserKeytab(hadoopSecurity.getUserKeytab());
        fb.setNamenodePrincipal(hadoopSecurity.getNamenodePrincipal());
        fb.setRmManagerPrincipal(hadoopSecurity.getRmManagerPrincipal());
    }

    fb.afterPropertiesSet();

    YarnConfiguration c = fb.getObject();
    c = postProcess(c);
    return c;
}

From source file:org.springframework.yarn.configuration.ConfigurationFactoryBean.java

License:Apache License

/**
 * Creates a configuration instance potentially using the existing one (passed as an argument - which can be null).
 *
 * @param existing/*www . ja v a  2  s. c  o m*/
 * @return configuration instance
 */
protected YarnConfiguration createConfiguration(Configuration existing) {
    return (existing != null ? new YarnConfiguration(existing) : new YarnConfiguration());
}

From source file:org.springframework.yarn.configuration.ConfigurationUtils.java

License:Apache License

/**
 * Creates a new {@link Configuration} based on the given arguments.
 *
 * @param original initial configuration to read from. May be null.
 * @param properties properties object to add to the newly created configuration. May be null.
 * @return newly created configuration based on the input parameters.
 *//*from  w  w w. j av a 2  s.co m*/
public static Configuration createFrom(Configuration original, Properties properties) {
    Configuration cfg = null;
    if (original != null) {
        cfg = (original instanceof YarnConfiguration ? new YarnConfiguration(original)
                : new Configuration(original));
    } else {
        cfg = new YarnConfiguration();
    }
    addProperties(cfg, properties);
    return cfg;
}

From source file:org.springframework.yarn.test.MiniYARNCluster.java

License:Apache License

@Override
public void init(Configuration conf) {
    super.init(conf instanceof YarnConfiguration ? conf : new YarnConfiguration(conf));
}

From source file:org.zuinnote.hadoop.office.example.MapReduceExcelInputIntegrationTest.java

License:Apache License

@BeforeAll
public static void oneTimeSetUp() throws IOException {
    // Create temporary directory for HDFS base and shutdownhook 
    // create temp directory
    tmpPath = Files.createTempDirectory(tmpPrefix);
    // create shutdown hook to remove temp files (=HDFS MiniCluster) after shutdown, may need to rethink to avoid many threads are created
    Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
        @Override/*from  w  w w .  ja  v a  2 s . c  om*/
        public void run() {
            try {
                Files.walkFileTree(tmpPath, new SimpleFileVisitor<java.nio.file.Path>() {

                    @Override
                    public FileVisitResult visitFile(java.nio.file.Path file, BasicFileAttributes attrs)
                            throws IOException {
                        Files.delete(file);
                        return FileVisitResult.CONTINUE;
                    }

                    @Override
                    public FileVisitResult postVisitDirectory(java.nio.file.Path dir, IOException e)
                            throws IOException {
                        if (e == null) {
                            Files.delete(dir);
                            return FileVisitResult.CONTINUE;
                        }
                        throw e;
                    }
                });
            } catch (IOException e) {
                throw new RuntimeException(
                        "Error temporary files in following path could not be deleted " + tmpPath, e);
            }
        }
    }));
    // Create Configuration
    Configuration conf = new Configuration();
    // create HDFS cluster
    File baseDir = new File(tmpPath.toString()).getAbsoluteFile();
    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
    MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
    dfsCluster = builder.numDataNodes(NOOFDATANODES).build();
    // create Yarn cluster
    YarnConfiguration clusterConf = new YarnConfiguration(conf);
    conf.set("fs.defaultFS", dfsCluster.getFileSystem().getUri().toString());
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
    miniCluster = new MiniMRYarnCluster(CLUSTERNAME, NOOFNODEMANAGERS, STARTTIMELINESERVER);
    miniCluster.init(conf);
    miniCluster.start();
}

From source file:oz.hadoop.yarn.api.core.AbstractContainer.java

License:Apache License

/**
 * /*from w w w .ja v a  2s  .  c  o m*/
 * @param applicationSpecification
 */
@SuppressWarnings("unchecked")
public AbstractContainer(PrimitiveImmutableTypeMap applicationSpecification) {
    if (logger.isInfoEnabled()) {
        logger.info("Creating " + this.getClass().getName());
    }

    this.yarnConfig = new YarnConfiguration(new Configuration());

    this.applicationSpecification = applicationSpecification;
    this.containerSpec = new PrimitiveImmutableTypeMap(
            (Map<String, Object>) this.applicationSpecification.get(YayaConstants.CONTAINER_SPEC));

    if (logger.isInfoEnabled()) {
        logger.info(
                "Application Container specification: " + PrintUtils.prettyMap(this.applicationSpecification));
    }
    if (logger.isTraceEnabled()) {
        logger.trace("SYSTEM PROPERTIES:\n" + PrintUtils.prettyMap(System.getProperties()));
        logger.trace("ENVIRONMENT VARIABLES:\n" + PrintUtils.prettyMap(System.getenv()));
    }
}