Example usage for org.apache.spark.launcher SparkLauncher addJar

Introduction

In this page you can find the example usage for org.apache.spark.launcher SparkLauncher addJar.

Prototype

@Override
    public SparkLauncher addJar(String jar) 

Source Link

Usage

From source file:com.uber.hoodie.cli.utils.SparkUtil.java

License:Apache License

/**
 * TODO: Need to fix a bunch of hardcoded stuff here eg: history server, spark distro
 *///from  ww  w  .j  a v a 2s .co m
public static SparkLauncher initLauncher(String propertiesFile) throws URISyntaxException {
    String currentJar = new File(
            SparkUtil.class.getProtectionDomain().getCodeSource().getLocation().toURI().getPath())
                    .getAbsolutePath();
    SparkLauncher sparkLauncher = new SparkLauncher().setAppResource(currentJar)
            .setMainClass(SparkMain.class.getName());

    if (StringUtils.isNotEmpty(propertiesFile)) {
        sparkLauncher.setPropertiesFile(propertiesFile);
    }

    File libDirectory = new File(new File(currentJar).getParent(), "lib");
    for (String library : libDirectory.list()) {
        sparkLauncher.addJar(new File(libDirectory, library).getAbsolutePath());
    }
    return sparkLauncher;
}

---

From source file:org.datacleaner.spark.ApplicationDriver.java

License:Open Source License

public SparkLauncher createSparkLauncher(File hadoopConfDir, String configurationHdfsPath, String jobHdfsPath)
        throws Exception {
    // mimic env. variables
    final Map<String, String> env = new HashMap<>();
    env.put("YARN_CONF_DIR", hadoopConfDir.getAbsolutePath());

    final SparkLauncher sparkLauncher = new SparkLauncher(env);

    sparkLauncher.setSparkHome(_sparkHome);
    sparkLauncher.setMaster("yarn-cluster");
    sparkLauncher.setAppName("DataCleaner");

    final MutableRef<String> primaryJar = new MutableRef<>();
    final List<String> jars = buildJarFiles(primaryJar);
    logger.info("Using JAR files: {}", jars);

    for (final String jar : jars) {
        sparkLauncher.addJar(jar);
    }/*w w w  .  j a v  a  2s . co  m*/
    sparkLauncher.setMainClass(Main.class.getName());

    // the primary jar is always the first argument
    sparkLauncher.addAppArgs(primaryJar.get());

    sparkLauncher.addAppArgs(toHdfsPath(configurationHdfsPath));
    sparkLauncher.addAppArgs(toHdfsPath(jobHdfsPath));

    return sparkLauncher;
}

---