Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hive.spark.client; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION; import com.google.common.base.Charsets; import com.google.common.base.Joiner; import com.google.common.base.Strings; import com.google.common.base.Preconditions; import com.google.common.base.Throwables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.io.Resources; import io.netty.channel.ChannelHandlerContext; import io.netty.util.concurrent.GenericFutureListener; import io.netty.util.concurrent.Promise; import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.io.Serializable; import java.io.Writer; import java.net.URI; import java.net.URL; import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.UUID; import java.util.concurrent.Future; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.shims.Utils; import org.apache.hadoop.security.SecurityUtil; import org.apache.hive.spark.client.rpc.Rpc; import org.apache.hive.spark.client.rpc.RpcConfiguration; import org.apache.hive.spark.client.rpc.RpcServer; import org.apache.spark.SparkContext; import org.apache.spark.SparkException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; class SparkClientImpl implements SparkClient { private static final long serialVersionUID = 1L; private static final Logger LOG = LoggerFactory.getLogger(SparkClientImpl.class); private static final long DEFAULT_SHUTDOWN_TIMEOUT = 10000; // In milliseconds private static final long MAX_ERR_LOG_LINES_FOR_RPC = 1000; private static final String OSX_TEST_OPTS = "SPARK_OSX_TEST_OPTS"; private static final String SPARK_HOME_ENV = "SPARK_HOME"; private static final String SPARK_HOME_KEY = "spark.home"; private static final String DRIVER_OPTS_KEY = "spark.driver.extraJavaOptions"; private static final String EXECUTOR_OPTS_KEY = "spark.executor.extraJavaOptions"; private static final String DRIVER_EXTRA_CLASSPATH = "spark.driver.extraClassPath"; private static final String EXECUTOR_EXTRA_CLASSPATH = "spark.executor.extraClassPath"; private final Map<String, String> conf; private final HiveConf hiveConf; private final Thread driverThread; private final Map<String, JobHandleImpl<?>> jobs; private final Rpc driverRpc; private final ClientProtocol protocol; private volatile boolean isAlive; SparkClientImpl(RpcServer rpcServer, Map<String, String> conf, HiveConf hiveConf) throws IOException, SparkException { this.conf = conf; this.hiveConf = hiveConf; this.jobs = Maps.newConcurrentMap(); String clientId = UUID.randomUUID().toString(); String secret = rpcServer.createSecret(); this.driverThread = startDriver(rpcServer, clientId, secret); this.protocol = new ClientProtocol(); try { // The RPC server will take care of timeouts here. this.driverRpc = rpcServer.registerClient(clientId, secret, protocol).get(); } catch (Throwable e) { String errorMsg = null; if (e.getCause() instanceof TimeoutException) { errorMsg = "Timed out waiting for client to connect.\nPossible reasons include network " + "issues, errors in remote driver or the cluster has no available resources, etc." + "\nPlease check YARN or Spark driver's logs for further information."; } else if (e.getCause() instanceof InterruptedException) { errorMsg = "Interruption occurred while waiting for client to connect.\nPossibly the Spark session is closed " + "such as in case of query cancellation." + "\nPlease refer to HiveServer2 logs for further information."; } else { errorMsg = "Error while waiting for client to connect."; } LOG.error(errorMsg, e); driverThread.interrupt(); try { driverThread.join(); } catch (InterruptedException ie) { // Give up. LOG.warn("Interrupted before driver thread was finished.", ie); } throw Throwables.propagate(e); } driverRpc.addListener(new Rpc.Listener() { @Override public void rpcClosed(Rpc rpc) { if (isAlive) { LOG.warn("Client RPC channel closed unexpectedly."); isAlive = false; } } }); isAlive = true; } @Override public <T extends Serializable> JobHandle<T> submit(Job<T> job) { return protocol.submit(job, Collections.<JobHandle.Listener<T>>emptyList()); } @Override public <T extends Serializable> JobHandle<T> submit(Job<T> job, List<JobHandle.Listener<T>> listeners) { return protocol.submit(job, listeners); } @Override public <T extends Serializable> Future<T> run(Job<T> job) { return protocol.run(job); } @Override public void stop() { if (isAlive) { isAlive = false; try { protocol.endSession(); } catch (Exception e) { LOG.warn("Exception while waiting for end session reply.", e); } finally { driverRpc.close(); } } long endTime = System.currentTimeMillis() + DEFAULT_SHUTDOWN_TIMEOUT; try { driverThread.join(DEFAULT_SHUTDOWN_TIMEOUT); } catch (InterruptedException ie) { LOG.debug("Interrupted before driver thread was finished."); } if (endTime - System.currentTimeMillis() <= 0) { LOG.warn("Timed out shutting down remote driver, interrupting..."); driverThread.interrupt(); } } @Override public Future<?> addJar(URI uri) { return run(new AddJarJob(uri.toString())); } @Override public Future<?> addFile(URI uri) { return run(new AddFileJob(uri.toString())); } @Override public Future<Integer> getExecutorCount() { return run(new GetExecutorCountJob()); } @Override public Future<Integer> getDefaultParallelism() { return run(new GetDefaultParallelismJob()); } @Override public boolean isActive() { return isAlive && driverRpc.isActive(); } void cancel(String jobId) { protocol.cancel(jobId); } private Thread startDriver(final RpcServer rpcServer, final String clientId, final String secret) throws IOException { Runnable runnable; final String serverAddress = rpcServer.getAddress(); final String serverPort = String.valueOf(rpcServer.getPort()); if (conf.containsKey(SparkClientFactory.CONF_KEY_IN_PROCESS)) { // Mostly for testing things quickly. Do not do this in production. // when invoked in-process it inherits the environment variables of the parent LOG.warn("!!!! Running remote driver in-process. !!!!"); runnable = new Runnable() { @Override public void run() { List<String> args = Lists.newArrayList(); args.add("--remote-host"); args.add(serverAddress); args.add("--remote-port"); args.add(serverPort); args.add("--client-id"); args.add(clientId); args.add("--secret"); args.add(secret); for (Map.Entry<String, String> e : conf.entrySet()) { args.add("--conf"); args.add(String.format("%s=%s", e.getKey(), conf.get(e.getKey()))); } try { RemoteDriver.main(args.toArray(new String[args.size()])); } catch (Exception e) { LOG.error("Error running driver.", e); } } }; } else { // If a Spark installation is provided, use the spark-submit script. Otherwise, call the // SparkSubmit class directly, which has some caveats (like having to provide a proper // version of Guava on the classpath depending on the deploy mode). String sparkHome = Strings.emptyToNull(conf.get(SPARK_HOME_KEY)); if (sparkHome == null) { sparkHome = Strings.emptyToNull(System.getenv(SPARK_HOME_ENV)); } if (sparkHome == null) { sparkHome = Strings.emptyToNull(System.getProperty(SPARK_HOME_KEY)); } String sparkLogDir = conf.get("hive.spark.log.dir"); if (sparkLogDir == null) { if (sparkHome == null) { sparkLogDir = "./target/"; } else { sparkLogDir = sparkHome + "/logs/"; } } String osxTestOpts = ""; if (Strings.nullToEmpty(System.getProperty("os.name")).toLowerCase().contains("mac")) { osxTestOpts = Strings.nullToEmpty(System.getenv(OSX_TEST_OPTS)); } String driverJavaOpts = Joiner.on(" ").skipNulls().join("-Dhive.spark.log.dir=" + sparkLogDir, osxTestOpts, conf.get(DRIVER_OPTS_KEY)); String executorJavaOpts = Joiner.on(" ").skipNulls().join("-Dhive.spark.log.dir=" + sparkLogDir, osxTestOpts, conf.get(EXECUTOR_OPTS_KEY)); // Create a file with all the job properties to be read by spark-submit. Change the // file's permissions so that only the owner can read it. This avoid having the // connection secret show up in the child process's command line. File properties = File.createTempFile("spark-submit.", ".properties"); if (!properties.setReadable(false) || !properties.setReadable(true, true)) { throw new IOException("Cannot change permissions of job properties file."); } properties.deleteOnExit(); Properties allProps = new Properties(); // first load the defaults from spark-defaults.conf if available try { URL sparkDefaultsUrl = Thread.currentThread().getContextClassLoader() .getResource("spark-defaults.conf"); if (sparkDefaultsUrl != null) { LOG.info("Loading spark defaults: " + sparkDefaultsUrl); allProps.load(new ByteArrayInputStream(Resources.toByteArray(sparkDefaultsUrl))); } } catch (Exception e) { String msg = "Exception trying to load spark-defaults.conf: " + e; throw new IOException(msg, e); } // then load the SparkClientImpl config for (Map.Entry<String, String> e : conf.entrySet()) { allProps.put(e.getKey(), conf.get(e.getKey())); } allProps.put(SparkClientFactory.CONF_CLIENT_ID, clientId); allProps.put(SparkClientFactory.CONF_KEY_SECRET, secret); allProps.put(DRIVER_OPTS_KEY, driverJavaOpts); allProps.put(EXECUTOR_OPTS_KEY, executorJavaOpts); String isTesting = conf.get("spark.testing"); if (isTesting != null && isTesting.equalsIgnoreCase("true")) { String hiveHadoopTestClasspath = Strings.nullToEmpty(System.getenv("HIVE_HADOOP_TEST_CLASSPATH")); if (!hiveHadoopTestClasspath.isEmpty()) { String extraDriverClasspath = Strings .nullToEmpty((String) allProps.get(DRIVER_EXTRA_CLASSPATH)); if (extraDriverClasspath.isEmpty()) { allProps.put(DRIVER_EXTRA_CLASSPATH, hiveHadoopTestClasspath); } else { extraDriverClasspath = extraDriverClasspath.endsWith(File.pathSeparator) ? extraDriverClasspath : extraDriverClasspath + File.pathSeparator; allProps.put(DRIVER_EXTRA_CLASSPATH, extraDriverClasspath + hiveHadoopTestClasspath); } String extraExecutorClasspath = Strings .nullToEmpty((String) allProps.get(EXECUTOR_EXTRA_CLASSPATH)); if (extraExecutorClasspath.isEmpty()) { allProps.put(EXECUTOR_EXTRA_CLASSPATH, hiveHadoopTestClasspath); } else { extraExecutorClasspath = extraExecutorClasspath.endsWith(File.pathSeparator) ? extraExecutorClasspath : extraExecutorClasspath + File.pathSeparator; allProps.put(EXECUTOR_EXTRA_CLASSPATH, extraExecutorClasspath + hiveHadoopTestClasspath); } } } Writer writer = new OutputStreamWriter(new FileOutputStream(properties), Charsets.UTF_8); try { allProps.store(writer, "Spark Context configuration"); } finally { writer.close(); } // Define how to pass options to the child process. If launching in client (or local) // mode, the driver options need to be passed directly on the command line. Otherwise, // SparkSubmit will take care of that for us. String master = conf.get("spark.master"); Preconditions.checkArgument(master != null, "spark.master is not defined."); String deployMode = conf.get("spark.submit.deployMode"); List<String> argv = Lists.newLinkedList(); if (sparkHome != null) { argv.add(new File(sparkHome, "bin/spark-submit").getAbsolutePath()); } else { LOG.info("No spark.home provided, calling SparkSubmit directly."); argv.add(new File(System.getProperty("java.home"), "bin/java").getAbsolutePath()); if (master.startsWith("local") || master.startsWith("mesos") || SparkClientUtilities.isYarnClientMode(master, deployMode) || master.startsWith("spark")) { String mem = conf.get("spark.driver.memory"); if (mem != null) { argv.add("-Xms" + mem); argv.add("-Xmx" + mem); } String cp = conf.get("spark.driver.extraClassPath"); if (cp != null) { argv.add("-classpath"); argv.add(cp); } String libPath = conf.get("spark.driver.extraLibPath"); if (libPath != null) { argv.add("-Djava.library.path=" + libPath); } String extra = conf.get(DRIVER_OPTS_KEY); if (extra != null) { for (String opt : extra.split("[ ]")) { if (!opt.trim().isEmpty()) { argv.add(opt.trim()); } } } } argv.add("org.apache.spark.deploy.SparkSubmit"); } if (SparkClientUtilities.isYarnClusterMode(master, deployMode)) { String executorCores = conf.get("spark.executor.cores"); if (executorCores != null) { argv.add("--executor-cores"); argv.add(executorCores); } String executorMemory = conf.get("spark.executor.memory"); if (executorMemory != null) { argv.add("--executor-memory"); argv.add(executorMemory); } String numOfExecutors = conf.get("spark.executor.instances"); if (numOfExecutors != null) { argv.add("--num-executors"); argv.add(numOfExecutors); } } // The options --principal/--keypad do not work with --proxy-user in spark-submit.sh // (see HIVE-15485, SPARK-5493, SPARK-19143), so Hive could only support doAs or // delegation token renewal, but not both. Since doAs is a more common case, if both // are needed, we choose to favor doAs. So when doAs is enabled, we use kinit command, // otherwise, we pass the principal/keypad to spark to support the token renewal for // long-running application. if ("kerberos".equals(hiveConf.get(HADOOP_SECURITY_AUTHENTICATION))) { String principal = SecurityUtil .getServerPrincipal(hiveConf.getVar(ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL), "0.0.0.0"); String keyTabFile = hiveConf.getVar(ConfVars.HIVE_SERVER2_KERBEROS_KEYTAB); if (StringUtils.isNotBlank(principal) && StringUtils.isNotBlank(keyTabFile)) { if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS)) { List<String> kinitArgv = Lists.newLinkedList(); kinitArgv.add("kinit"); kinitArgv.add(principal); kinitArgv.add("-k"); kinitArgv.add("-t"); kinitArgv.add(keyTabFile + ";"); kinitArgv.addAll(argv); argv = kinitArgv; } else { // if doAs is not enabled, we pass the principal/keypad to spark-submit in order to // support the possible delegation token renewal in Spark argv.add("--principal"); argv.add(principal); argv.add("--keytab"); argv.add(keyTabFile); } } } if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS)) { try { String currentUser = Utils.getUGI().getShortUserName(); // do not do impersonation in CLI mode if (!currentUser.equals(System.getProperty("user.name"))) { LOG.info("Attempting impersonation of " + currentUser); argv.add("--proxy-user"); argv.add(currentUser); } } catch (Exception e) { String msg = "Cannot obtain username: " + e; throw new IllegalStateException(msg, e); } } argv.add("--properties-file"); argv.add(properties.getAbsolutePath()); argv.add("--class"); argv.add(RemoteDriver.class.getName()); String jar = "spark-internal"; if (SparkContext.jarOfClass(this.getClass()).isDefined()) { jar = SparkContext.jarOfClass(this.getClass()).get(); } argv.add(jar); argv.add("--remote-host"); argv.add(serverAddress); argv.add("--remote-port"); argv.add(serverPort); //hive.spark.* keys are passed down to the RemoteDriver via --conf, //as --properties-file contains the spark.* keys that are meant for SparkConf object. for (String hiveSparkConfKey : RpcConfiguration.HIVE_SPARK_RSC_CONFIGS) { String value = RpcConfiguration.getValue(hiveConf, hiveSparkConfKey); argv.add("--conf"); argv.add(String.format("%s=%s", hiveSparkConfKey, value)); } String cmd = Joiner.on(" ").join(argv); LOG.info("Running client driver with argv: {}", cmd); ProcessBuilder pb = new ProcessBuilder("sh", "-c", cmd); // Prevent hive configurations from being visible in Spark. pb.environment().remove("HIVE_HOME"); pb.environment().remove("HIVE_CONF_DIR"); // Add credential provider password to the child process's environment // In case of Spark the credential provider location is provided in the jobConf when the job is submitted String password = getSparkJobCredentialProviderPassword(); if (password != null) { pb.environment().put(Constants.HADOOP_CREDENTIAL_PASSWORD_ENVVAR, password); } if (isTesting != null) { pb.environment().put("SPARK_TESTING", isTesting); } final Process child = pb.start(); String threadName = Thread.currentThread().getName(); final List<String> childErrorLog = Collections.synchronizedList(new ArrayList<String>()); redirect("RemoteDriver-stdout-redir-" + threadName, new Redirector(child.getInputStream())); redirect("RemoteDriver-stderr-redir-" + threadName, new Redirector(child.getErrorStream(), childErrorLog)); runnable = new Runnable() { @Override public void run() { try { int exitCode = child.waitFor(); if (exitCode != 0) { StringBuilder errStr = new StringBuilder(); synchronized (childErrorLog) { Iterator iter = childErrorLog.iterator(); while (iter.hasNext()) { errStr.append(iter.next()); errStr.append('\n'); } } LOG.warn("Child process exited with code {}", exitCode); rpcServer.cancelClient(clientId, "Child process (spark-submit) exited before connecting back with error log " + errStr.toString()); } } catch (InterruptedException ie) { LOG.warn( "Thread waiting on the child process (spark-submit) is interrupted, killing the child process."); rpcServer.cancelClient(clientId, "Thread waiting on the child porcess (spark-submit) is interrupted"); Thread.interrupted(); child.destroy(); } catch (Exception e) { String errMsg = "Exception while waiting for child process (spark-submit)"; LOG.warn(errMsg, e); rpcServer.cancelClient(clientId, errMsg); } } }; } Thread thread = new Thread(runnable); thread.setDaemon(true); thread.setName("Driver"); thread.start(); return thread; } private String getSparkJobCredentialProviderPassword() { if (conf.containsKey("spark.yarn.appMasterEnv.HADOOP_CREDSTORE_PASSWORD")) { return conf.get("spark.yarn.appMasterEnv.HADOOP_CREDSTORE_PASSWORD"); } else if (conf.containsKey("spark.executorEnv.HADOOP_CREDSTORE_PASSWORD")) { return conf.get("spark.executorEnv.HADOOP_CREDSTORE_PASSWORD"); } return null; } private void redirect(String name, Redirector redirector) { Thread thread = new Thread(redirector); thread.setName(name); thread.setDaemon(true); thread.start(); } private class ClientProtocol extends BaseProtocol { <T extends Serializable> JobHandleImpl<T> submit(Job<T> job, List<JobHandle.Listener<T>> listeners) { final String jobId = UUID.randomUUID().toString(); final Promise<T> promise = driverRpc.createPromise(); final JobHandleImpl<T> handle = new JobHandleImpl<T>(SparkClientImpl.this, promise, jobId, listeners); jobs.put(jobId, handle); final io.netty.util.concurrent.Future<Void> rpc = driverRpc.call(new JobRequest(jobId, job)); LOG.debug("Send JobRequest[{}].", jobId); // Link the RPC and the promise so that events from one are propagated to the other as // needed. rpc.addListener(new GenericFutureListener<io.netty.util.concurrent.Future<Void>>() { @Override public void operationComplete(io.netty.util.concurrent.Future<Void> f) { if (f.isSuccess()) { // If the spark job finishes before this listener is called, the QUEUED status will not be set handle.changeState(JobHandle.State.QUEUED); } else if (!promise.isDone()) { promise.setFailure(f.cause()); } } }); promise.addListener(new GenericFutureListener<Promise<T>>() { @Override public void operationComplete(Promise<T> p) { if (jobId != null) { jobs.remove(jobId); } if (p.isCancelled() && !rpc.isDone()) { rpc.cancel(true); } } }); return handle; } <T extends Serializable> Future<T> run(Job<T> job) { @SuppressWarnings("unchecked") final io.netty.util.concurrent.Future<T> rpc = (io.netty.util.concurrent.Future<T>) driverRpc .call(new SyncJobRequest(job), Serializable.class); return rpc; } void cancel(String jobId) { driverRpc.call(new CancelJob(jobId)); } Future<?> endSession() { return driverRpc.call(new EndSession()); } private void handle(ChannelHandlerContext ctx, Error msg) { LOG.warn("Error reported from remote driver.", msg.cause); } private void handle(ChannelHandlerContext ctx, JobMetrics msg) { JobHandleImpl<?> handle = jobs.get(msg.jobId); if (handle != null) { handle.getMetrics().addMetrics(msg.sparkJobId, msg.stageId, msg.taskId, msg.metrics); } else { LOG.warn("Received metrics for unknown job {}", msg.jobId); } } private void handle(ChannelHandlerContext ctx, JobResult msg) { JobHandleImpl<?> handle = jobs.remove(msg.id); if (handle != null) { LOG.info("Received result for {}", msg.id); handle.setSparkCounters(msg.sparkCounters); Throwable error = msg.error != null ? new SparkException(msg.error) : null; if (error == null) { handle.setSuccess(msg.result); } else { handle.setFailure(error); } } else { LOG.warn("Received result for unknown job {}", msg.id); } } private void handle(ChannelHandlerContext ctx, JobStarted msg) { JobHandleImpl<?> handle = jobs.get(msg.id); if (handle != null) { handle.changeState(JobHandle.State.STARTED); } else { LOG.warn("Received event for unknown job {}", msg.id); } } private void handle(ChannelHandlerContext ctx, JobSubmitted msg) { JobHandleImpl<?> handle = jobs.get(msg.clientJobId); if (handle != null) { LOG.info("Received spark job ID: {} for {}", msg.sparkJobId, msg.clientJobId); handle.addSparkJobId(msg.sparkJobId); } else { LOG.warn("Received spark job ID: {} for unknown job {}", msg.sparkJobId, msg.clientJobId); } } } private class Redirector implements Runnable { private final BufferedReader in; private List<String> errLogs; private int numErrLogLines = 0; Redirector(InputStream in) { this.in = new BufferedReader(new InputStreamReader(in)); } Redirector(InputStream in, List<String> errLogs) { this.in = new BufferedReader(new InputStreamReader(in)); this.errLogs = errLogs; } @Override public void run() { try { String line = null; while ((line = in.readLine()) != null) { LOG.info(line); if (errLogs != null) { if (numErrLogLines++ < MAX_ERR_LOG_LINES_FOR_RPC) { errLogs.add(line); } } } } catch (IOException e) { if (isAlive) { LOG.warn("I/O error in redirector thread.", e); } else { // When stopping the remote driver the process might be destroyed during reading from the stream. // We should not log the related exceptions in a visible level as they might mislead the user. LOG.debug("I/O error in redirector thread while stopping the remote driver", e); } } catch (Exception e) { LOG.warn("Error in redirector thread.", e); } } } private static class AddJarJob implements Job<Serializable> { private static final long serialVersionUID = 1L; private final String path; AddJarJob() { this(null); } AddJarJob(String path) { this.path = path; } @Override public Serializable call(JobContext jc) throws Exception { jc.sc().addJar(path); // Following remote job may refer to classes in this jar, and the remote job would be executed // in a different thread, so we add this jar path to JobContext for further usage. jc.getAddedJars().put(path, System.currentTimeMillis()); return null; } } private static class AddFileJob implements Job<Serializable> { private static final long serialVersionUID = 1L; private final String path; AddFileJob() { this(null); } AddFileJob(String path) { this.path = path; } @Override public Serializable call(JobContext jc) throws Exception { jc.sc().addFile(path); return null; } } private static class GetExecutorCountJob implements Job<Integer> { private static final long serialVersionUID = 1L; @Override public Integer call(JobContext jc) throws Exception { // minus 1 here otherwise driver is also counted as an executor int count = jc.sc().sc().getExecutorMemoryStatus().size() - 1; return Integer.valueOf(count); } } private static class GetDefaultParallelismJob implements Job<Integer> { private static final long serialVersionUID = 1L; @Override public Integer call(JobContext jc) throws Exception { return jc.sc().sc().defaultParallelism(); } } }