Example usage for org.apache.spark.launcher SparkLauncher addFile

Introduction

In this page you can find the example usage for org.apache.spark.launcher SparkLauncher addFile.

Prototype

@Override
    public SparkLauncher addFile(String file) 

Source Link

Usage

From source file:org.cripac.isee.vpe.ctrl.SystemPropertyCenter.java

License:Open Source License

SparkLauncher GetSparkLauncher(String appName) throws IOException, NoAppSpecifiedException {
    SparkLauncher launcher = new SparkLauncher().setAppResource(jarPath)
            .setMainClass(AppManager.getMainClassName(appName)).setMaster(sparkMaster).setAppName(appName)
            .setVerbose(verbose).addFile(ConfManager.getConcatCfgFilePathList(","))
            .setConf(SparkLauncher.DRIVER_MEMORY, driverMem).setConf(SparkLauncher.EXECUTOR_MEMORY, executorMem)
            .setConf(SparkLauncher.CHILD_PROCESS_LOGGER_NAME, appName)
            .setConf(SparkLauncher.EXECUTOR_CORES, "" + executorCores)
            .setConf("spark.driver.extraJavaOptions", "-Dlog4j.configuration=log4j.properties")
            .setConf("spark.executor.extraJavaOptions", "-Dlog4j.configuration=log4j.properties")
            .setConf("spark.yarn.am.nodeLabelExpression", yarnAmNodeLabelExpression)
            .addSparkArg("--driver-cores", "" + driverCores).addSparkArg("--num-executors", "" + numExecutors)
            .addSparkArg("--total-executor-cores", "" + totalExecutorCores).addSparkArg("--queue", hadoopQueue)
            .addAppArgs(getArgs());// ww w.  j a v  a 2 s  .c o  m
    if (sparkConfFilePath != null) {
        if (new File(sparkConfFilePath).exists()) {
            launcher = launcher.setPropertiesFile(sparkConfFilePath);
        } else {
            logger.warn("Spark configuration file " + sparkConfFilePath + " does not exist!");
        }
    }
    if (log4jPropFilePath != null) {
        if (new File(log4jPropFilePath).exists()) {
            launcher = launcher.addFile(log4jPropFilePath);
        } else {
            logger.warn("Loj4j configuration file " + log4jPropFilePath + " does not exist!");
        }
    }
    if (sysPropFilePath != null) {
        if (new File(sysPropFilePath).exists()) {
            launcher = launcher.addFile(sysPropFilePath);
        } else {
            logger.warn("System configuration file " + sysPropFilePath + " does not exist!");
        }
        launcher = launcher.addFile(sysPropFilePath);
    }
    if (appPropFilePath != null) {
        if (new File(appPropFilePath).exists()) {
            launcher = launcher.addFile(appPropFilePath);
        } else {
            logger.warn("App configuration file " + appPropFilePath + " does not exist!");
        }
    }
    return launcher;
}

---