Java tutorial
package com.thinkbiganalytics.nifi.pyspark.core; /*- * #%L * thinkbig-nifi-spark-processors * %% * Copyright (C) 2017 ThinkBig Analytics * %% * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * #L% */ import com.thinkbiganalytics.nifi.processor.AbstractNiFiProcessor; import com.thinkbiganalytics.nifi.pyspark.utils.PySparkUtils; import com.thinkbiganalytics.nifi.security.ApplySecurityPolicy; import com.thinkbiganalytics.nifi.security.KerberosProperties; import com.thinkbiganalytics.nifi.security.SecurityUtil; import com.thinkbiganalytics.nifi.security.SpringSecurityContextLoader; import com.thinkbiganalytics.nifi.util.InputStreamReaderRunnable; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.nifi.annotation.behavior.EventDriven; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.components.ValidationContext; import org.apache.nifi.components.ValidationResult; import org.apache.nifi.components.Validator; import org.apache.nifi.flowfile.FlowFile; import org.apache.nifi.flowfile.attributes.CoreAttributes; import org.apache.nifi.logging.ComponentLog; import org.apache.nifi.logging.LogLevel; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessSession; import org.apache.nifi.processor.ProcessorInitializationContext; import org.apache.nifi.processor.Relationship; import org.apache.nifi.processor.exception.ProcessException; import org.apache.nifi.processor.util.StandardValidators; import org.apache.spark.launcher.SparkLauncher; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; import javax.annotation.Nonnull; /** * A NiFi processor to execute a PySpark job */ @EventDriven @Tags({ "spark", "thinkbig", "pyspark" }) @CapabilityDescription("Execute a PySpark job.") public class ExecutePySpark extends AbstractNiFiProcessor { /* Processor properties */ public static final PropertyDescriptor HADOOP_CONFIGURATION_RESOURCES = new PropertyDescriptor.Builder() .name("Hadoop Configuration Resources") .description( "A file or comma separated list of files which contains the Hadoop file system configuration. Without this, Hadoop " + "will search the classpath for a 'core-site.xml' and 'hdfs-site.xml' file or will revert to a default configuration. " + "NOTE: This value is also required for a Kerberized cluster.") .required(false).addValidator(multipleFilesExistValidator()).build(); public static final PropertyDescriptor PYSPARK_APP_FILE = new PropertyDescriptor.Builder() .name("PySpark App File") .description("Full path for PySpark application file (having Python code to be executed)") .required(true).addValidator(new StandardValidators.FileExistsValidator(true)) .expressionLanguageSupported(true).build(); public static final PropertyDescriptor PYSPARK_APP_ARGS = new PropertyDescriptor.Builder() .name("PySpark App Args") .description("Comma separated arguments to be passed to the PySpark application. " + "NOTE: Ensure that no spaces are present between the comma separated arguments.") .required(false).addValidator(StandardValidators.NON_EMPTY_VALIDATOR).expressionLanguageSupported(true) .build(); public static final PropertyDescriptor PYSPARK_APP_NAME = new PropertyDescriptor.Builder() .name("PySpark App Name").description("A name for the PySpark application").required(true) .addValidator(StandardValidators.NON_EMPTY_VALIDATOR).expressionLanguageSupported(true) .defaultValue("PySpark-App").build(); public static final PropertyDescriptor PYSPARK_ADDITIONAL_FILES = new PropertyDescriptor.Builder() .name("Additional Python files/zips/eggs") .description( "(Comma separated) Full path for additional Python files/zips/eggs to be submitted with the application. " + "NOTE: Ensure that no spaces are present between the comma separated file locations.") .required(false).addValidator(multipleFilesExistValidator()).expressionLanguageSupported(true).build(); public static final PropertyDescriptor SPARK_MASTER = new PropertyDescriptor.Builder().name("Spark Master") .description("The Spark master. NOTE: Please ensure that you have not set this in your application.") .required(true).defaultValue("local").addValidator(StandardValidators.NON_EMPTY_VALIDATOR) .expressionLanguageSupported(true).build(); public static final PropertyDescriptor SPARK_YARN_DEPLOY_MODE = new PropertyDescriptor.Builder() .name("Spark YARN Deploy Mode") .description("The deploy mode for YARN master (client, cluster). Only applicable for yarn mode. " + "NOTE: Please ensure that you have not set this in your application.") .required(false).defaultValue("client").addValidator(StandardValidators.NON_EMPTY_VALIDATOR) .expressionLanguageSupported(true).build(); public static final PropertyDescriptor YARN_QUEUE = new PropertyDescriptor.Builder().name("YARN Queue") .description("The name of the YARN queue to which the job is submitted. Only applicable for yarn mode.") .required(false).addValidator(StandardValidators.NON_EMPTY_VALIDATOR).expressionLanguageSupported(true) .build(); public static final PropertyDescriptor SPARK_HOME = new PropertyDescriptor.Builder().name("Spark Home") .description("Spark installation location").required(true) .defaultValue("/usr/hdp/current/spark-client/") .addValidator(new StandardValidators.DirectoryExistsValidator(true, false)) .expressionLanguageSupported(true).build(); public static final PropertyDescriptor DRIVER_MEMORY = new PropertyDescriptor.Builder().name("Driver Memory") .description( "Amount of memory (RAM) to allocate to the driver (e.g. 512m, 2g). Consider cluster capacity when setting value.") .required(true).defaultValue("512m").addValidator(StandardValidators.NON_EMPTY_VALIDATOR) .expressionLanguageSupported(true).build(); public static final PropertyDescriptor EXECUTOR_MEMORY = new PropertyDescriptor.Builder() .name("Executor Memory") .description( "Amount of memory (RAM) to allocate to an executor (e.g. 512m, 2g). Consider cluster capacity when setting value.") .required(true).defaultValue("512m").addValidator(StandardValidators.NON_EMPTY_VALIDATOR) .expressionLanguageSupported(true).build(); public static final PropertyDescriptor EXECUTOR_INSTANCES = new PropertyDescriptor.Builder() .name("Executor Instances") .description( "The number of executors to use for job execution. Consider cluster capacity when setting value.") .required(true).defaultValue("1").addValidator(StandardValidators.createLongValidator(1L, 1000L, true)) .expressionLanguageSupported(true).build(); public static final PropertyDescriptor EXECUTOR_CORES = new PropertyDescriptor.Builder().name("Executor Cores") .description( "The number of CPU cores to be used on each executor. Consider cluster capacity when setting value.") .required(true).defaultValue("1").addValidator(StandardValidators.createLongValidator(1L, 100L, true)) .expressionLanguageSupported(true).build(); public static final PropertyDescriptor NETWORK_TIMEOUT = new PropertyDescriptor.Builder() .name("Network Timeout") .description("Default timeout for all network interactions. " + "This config will be used in place of spark.core.connection.ack.wait.timeout, " + "spark.akka.timeout, spark.storage.blockManagerSlaveTimeoutMs, " + "spark.shuffle.io.connectionTimeout, spark.rpc.askTimeout " + "or spark.rpc.lookupTimeout if they are not configured.") .required(true).defaultValue("120s").addValidator(StandardValidators.NON_EMPTY_VALIDATOR) .expressionLanguageSupported(true).build(); public static final PropertyDescriptor ADDITIONAL_SPARK_CONFIG_OPTIONS = new PropertyDescriptor.Builder() .name("Additional Spark Configuration") .description("Additional configuration options to pass to the Spark job. " + "These would be key=value pairs separated by comma. " + "Note that the configuration option would start with 'spark.' " + "e.g. spark.ui.port=4040 " + "NOTE: Ensure that no spaces are present between the comma-separated key=value pairs.") .required(false).addValidator(StandardValidators.NON_EMPTY_VALIDATOR).expressionLanguageSupported(true) .build(); /* Processor relationships */ public static final Relationship REL_SUCCESS = new Relationship.Builder().name("success") .description("PySpark job execution success").build(); public static final Relationship REL_FAILURE = new Relationship.Builder().name("failure") .description("PySpark job execution failure").build(); /* Spark configuration */ private static final String CONFIG_PROP_SPARK_YARN_KEYTAB = "spark.yarn.keytab"; private static final String CONFIG_PROP_SPARK_YARN_PRINCIPAL = "spark.yarn.principal"; private static final String CONFIG_PROP_SPARK_NETWORK_TIMEOUT = "spark.network.timeout"; private static final String CONFIG_PROP_SPARK_YARN_QUEUE = "spark.yarn.queue"; private static final String CONFIG_PROP_SPARK_EXECUTOR_INSTANCES = "spark.executor.instances"; /* Properties for Kerberos service keytab and principal */ private PropertyDescriptor KERBEROS_KEYTAB; private PropertyDescriptor KERBEROS_PRINCIPAL; private List<PropertyDescriptor> properties; private Set<Relationship> relationships; /* Validates that one or more files exist, as specified in a single property (comma-separated values) */ public static Validator multipleFilesExistValidator() { return new Validator() { @Override public ValidationResult validate(String subject, String input, ValidationContext context) { try { final String[] files = input.split(","); for (String filename : files) { try { final File file = new File(filename.trim()); if (!file.exists()) { final String message = "file " + filename + " does not exist."; return new ValidationResult.Builder().subject(this.getClass().getSimpleName()) .input(input).valid(false).explanation(message).build(); } else if (!file.isFile()) { final String message = filename + " is not a file."; return new ValidationResult.Builder().subject(this.getClass().getSimpleName()) .input(input).valid(false).explanation(message).build(); } else if (!file.canRead()) { final String message = "could not read " + filename; return new ValidationResult.Builder().subject(this.getClass().getSimpleName()) .input(input).valid(false).explanation(message).build(); } } catch (SecurityException e) { final String message = "unable to access " + filename + " due to " + e.getMessage(); return new ValidationResult.Builder().subject(this.getClass().getSimpleName()) .input(input).valid(false).explanation(message).build(); } } } catch (Exception e) { return new ValidationResult.Builder().subject(this.getClass().getSimpleName()).input(input) .valid(false) .explanation( "error evaluating value. Please sure that value is provided as file1,file2,file3 and so on. " + "Also, the files should exist and be readable.") .build(); } return new ValidationResult.Builder().subject(this.getClass().getSimpleName()).input(input) .valid(true).build(); } }; } @Override protected void init(@Nonnull final ProcessorInitializationContext context) { super.init(context); /* Create Kerberos properties */ final SpringSecurityContextLoader securityContextLoader = SpringSecurityContextLoader.create(context); final KerberosProperties kerberosProperties = securityContextLoader.getKerberosProperties(); KERBEROS_KEYTAB = kerberosProperties.createKerberosKeytabProperty(); KERBEROS_PRINCIPAL = kerberosProperties.createKerberosPrincipalProperty(); /* Create list of properties */ final List<PropertyDescriptor> properties = new ArrayList<>(); properties.add(KERBEROS_PRINCIPAL); properties.add(KERBEROS_KEYTAB); properties.add(HADOOP_CONFIGURATION_RESOURCES); properties.add(PYSPARK_APP_FILE); properties.add(PYSPARK_APP_ARGS); properties.add(PYSPARK_APP_NAME); properties.add(PYSPARK_ADDITIONAL_FILES); properties.add(SPARK_MASTER); properties.add(SPARK_YARN_DEPLOY_MODE); properties.add(YARN_QUEUE); properties.add(SPARK_HOME); properties.add(DRIVER_MEMORY); properties.add(EXECUTOR_MEMORY); properties.add(EXECUTOR_INSTANCES); properties.add(EXECUTOR_CORES); properties.add(NETWORK_TIMEOUT); properties.add(ADDITIONAL_SPARK_CONFIG_OPTIONS); this.properties = Collections.unmodifiableList(properties); /* Create list of relationships */ final Set<Relationship> relationships = new HashSet<>(); relationships.add(REL_SUCCESS); relationships.add(REL_FAILURE); this.relationships = Collections.unmodifiableSet(relationships); } @Override protected List<PropertyDescriptor> getSupportedPropertyDescriptors() { return properties; } @Override public Set<Relationship> getRelationships() { return relationships; } @Override public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { final ComponentLog logger = getLog(); FlowFile flowFile = session.get(); if (flowFile == null) { flowFile = session.create(); logger.info("Created a flow file having uuid: {}", new Object[] { flowFile.getAttribute(CoreAttributes.UUID.key()) }); } else { logger.info("Using an existing flow file having uuid: {}", new Object[] { flowFile.getAttribute(CoreAttributes.UUID.key()) }); } try { final String kerberosPrincipal = context.getProperty(KERBEROS_PRINCIPAL).getValue(); final String kerberosKeyTab = context.getProperty(KERBEROS_KEYTAB).getValue(); final String hadoopConfigurationResources = context.getProperty(HADOOP_CONFIGURATION_RESOURCES) .getValue(); final String pySparkAppFile = context.getProperty(PYSPARK_APP_FILE) .evaluateAttributeExpressions(flowFile).getValue(); final String pySparkAppArgs = context.getProperty(PYSPARK_APP_ARGS) .evaluateAttributeExpressions(flowFile).getValue(); final String pySparkAppName = context.getProperty(PYSPARK_APP_NAME) .evaluateAttributeExpressions(flowFile).getValue(); final String pySparkAdditionalFiles = context.getProperty(PYSPARK_ADDITIONAL_FILES) .evaluateAttributeExpressions(flowFile).getValue(); final String sparkMaster = context.getProperty(SPARK_MASTER).evaluateAttributeExpressions(flowFile) .getValue().trim().toLowerCase(); final String sparkYarnDeployMode = context.getProperty(SPARK_YARN_DEPLOY_MODE) .evaluateAttributeExpressions(flowFile).getValue(); final String yarnQueue = context.getProperty(YARN_QUEUE).evaluateAttributeExpressions(flowFile) .getValue(); final String sparkHome = context.getProperty(SPARK_HOME).evaluateAttributeExpressions(flowFile) .getValue(); final String driverMemory = context.getProperty(DRIVER_MEMORY).evaluateAttributeExpressions(flowFile) .getValue(); final String executorMemory = context.getProperty(EXECUTOR_MEMORY) .evaluateAttributeExpressions(flowFile).getValue(); final String executorInstances = context.getProperty(EXECUTOR_INSTANCES) .evaluateAttributeExpressions(flowFile).getValue(); final String executorCores = context.getProperty(EXECUTOR_CORES).evaluateAttributeExpressions(flowFile) .getValue(); final String networkTimeout = context.getProperty(NETWORK_TIMEOUT) .evaluateAttributeExpressions(flowFile).getValue(); final String additionalSparkConfigOptions = context.getProperty(ADDITIONAL_SPARK_CONFIG_OPTIONS) .evaluateAttributeExpressions(flowFile).getValue(); PySparkUtils pySparkUtils = new PySparkUtils(); /* Get app arguments */ String[] pySparkAppArgsArray = null; if (!StringUtils.isEmpty(pySparkAppArgs)) { pySparkAppArgsArray = pySparkUtils.getCsvValuesAsArray(pySparkAppArgs); logger.info("Provided application arguments: {}", new Object[] { pySparkUtils.getCsvStringFromArray(pySparkAppArgsArray) }); } /* Get additional python files */ String[] pySparkAdditionalFilesArray = null; if (!StringUtils.isEmpty(pySparkAdditionalFiles)) { pySparkAdditionalFilesArray = pySparkUtils.getCsvValuesAsArray(pySparkAdditionalFiles); logger.info("Provided python files: {}", new Object[] { pySparkUtils.getCsvStringFromArray(pySparkAdditionalFilesArray) }); } /* Get additional config key-value pairs */ String[] additionalSparkConfigOptionsArray = null; if (!StringUtils.isEmpty(additionalSparkConfigOptions)) { additionalSparkConfigOptionsArray = pySparkUtils.getCsvValuesAsArray(additionalSparkConfigOptions); logger.info("Provided spark config options: {}", new Object[] { pySparkUtils.getCsvStringFromArray(additionalSparkConfigOptionsArray) }); } /* Determine if Kerberos is enabled */ boolean kerberosEnabled = false; if (!StringUtils.isEmpty(kerberosPrincipal) && !StringUtils.isEmpty(kerberosKeyTab) && !StringUtils.isEmpty(hadoopConfigurationResources)) { kerberosEnabled = true; logger.info("Kerberos is enabled"); } /* For Kerberized cluster, attempt user authentication */ if (kerberosEnabled) { logger.info("Attempting user authentication for Kerberos"); ApplySecurityPolicy applySecurityObject = new ApplySecurityPolicy(); Configuration configuration; try { logger.info("Getting Hadoop configuration from " + hadoopConfigurationResources); configuration = ApplySecurityPolicy.getConfigurationFromResources(hadoopConfigurationResources); if (SecurityUtil.isSecurityEnabled(configuration)) { logger.info("Security is enabled"); if (kerberosPrincipal.equals("") && kerberosKeyTab.equals("")) { logger.error( "Kerberos Principal and Keytab provided with empty values for a Kerberized cluster."); session.transfer(flowFile, REL_FAILURE); return; } try { logger.info("User authentication initiated"); boolean authenticationStatus = applySecurityObject.validateUserWithKerberos(logger, hadoopConfigurationResources, kerberosPrincipal, kerberosKeyTab); if (authenticationStatus) { logger.info("User authenticated successfully."); } else { logger.error("User authentication failed."); session.transfer(flowFile, REL_FAILURE); return; } } catch (Exception unknownException) { logger.error("Unknown exception occurred while validating user :" + unknownException.getMessage()); session.transfer(flowFile, REL_FAILURE); return; } } } catch (IOException e1) { logger.error("Unknown exception occurred while authenticating user :" + e1.getMessage()); session.transfer(flowFile, REL_FAILURE); return; } } /* Build and launch PySpark Job */ logger.info("Configuring PySpark job for execution"); SparkLauncher pySparkLauncher = new SparkLauncher().setAppResource(pySparkAppFile); logger.info("PySpark app file set to: {}", new Object[] { pySparkAppFile }); if (pySparkAppArgsArray != null && pySparkAppArgsArray.length > 0) { pySparkLauncher = pySparkLauncher.addAppArgs(pySparkAppArgsArray); logger.info("App arguments set to: {}", new Object[] { pySparkUtils.getCsvStringFromArray(pySparkAppArgsArray) }); } pySparkLauncher = pySparkLauncher.setAppName(pySparkAppName).setMaster(sparkMaster); logger.info("App name set to: {}", new Object[] { pySparkAppName }); logger.info("Spark master set to: {}", new Object[] { sparkMaster }); if (pySparkAdditionalFilesArray != null && pySparkAdditionalFilesArray.length > 0) { for (String pySparkAdditionalFile : pySparkAdditionalFilesArray) { pySparkLauncher = pySparkLauncher.addPyFile(pySparkAdditionalFile); logger.info("Additional python file set to: {}", new Object[] { pySparkAdditionalFile }); } } if (sparkMaster.equals("yarn")) { pySparkLauncher = pySparkLauncher.setDeployMode(sparkYarnDeployMode); logger.info("YARN deploy mode set to: {}", new Object[] { sparkYarnDeployMode }); } pySparkLauncher = pySparkLauncher.setSparkHome(sparkHome) .setConf(SparkLauncher.DRIVER_MEMORY, driverMemory) .setConf(SparkLauncher.EXECUTOR_MEMORY, executorMemory) .setConf(CONFIG_PROP_SPARK_EXECUTOR_INSTANCES, executorInstances) .setConf(SparkLauncher.EXECUTOR_CORES, executorCores) .setConf(CONFIG_PROP_SPARK_NETWORK_TIMEOUT, networkTimeout); logger.info("Spark home set to: {} ", new Object[] { sparkHome }); logger.info("Driver memory set to: {} ", new Object[] { driverMemory }); logger.info("Executor memory set to: {} ", new Object[] { executorMemory }); logger.info("Executor instances set to: {} ", new Object[] { executorInstances }); logger.info("Executor cores set to: {} ", new Object[] { executorCores }); logger.info("Network timeout set to: {} ", new Object[] { networkTimeout }); if (kerberosEnabled) { pySparkLauncher = pySparkLauncher.setConf(CONFIG_PROP_SPARK_YARN_PRINCIPAL, kerberosPrincipal); pySparkLauncher = pySparkLauncher.setConf(CONFIG_PROP_SPARK_YARN_KEYTAB, kerberosKeyTab); logger.info("Kerberos principal set to: {} ", new Object[] { kerberosPrincipal }); logger.info("Kerberos keytab set to: {} ", new Object[] { kerberosKeyTab }); } if (!StringUtils.isEmpty(yarnQueue)) { pySparkLauncher = pySparkLauncher.setConf(CONFIG_PROP_SPARK_YARN_QUEUE, yarnQueue); logger.info("YARN queue set to: {} ", new Object[] { yarnQueue }); } if (additionalSparkConfigOptionsArray != null && additionalSparkConfigOptionsArray.length > 0) { for (String additionalSparkConfigOption : additionalSparkConfigOptionsArray) { String[] confKeyValue = additionalSparkConfigOption.split("="); if (confKeyValue.length == 2) { pySparkLauncher = pySparkLauncher.setConf(confKeyValue[0], confKeyValue[1]); logger.info("Spark additional config option set to: {}={}", new Object[] { confKeyValue[0], confKeyValue[1] }); } } } logger.info("Starting execution of PySpark job"); Process pySparkProcess = pySparkLauncher.launch(); InputStreamReaderRunnable inputStreamReaderRunnable = new InputStreamReaderRunnable(LogLevel.INFO, logger, pySparkProcess.getInputStream()); Thread inputThread = new Thread(inputStreamReaderRunnable, "stream input"); inputThread.start(); InputStreamReaderRunnable errorStreamReaderRunnable = new InputStreamReaderRunnable(LogLevel.INFO, logger, pySparkProcess.getErrorStream()); Thread errorThread = new Thread(errorStreamReaderRunnable, "stream error"); errorThread.start(); logger.info("Waiting for PySpark job to complete"); int exitCode = pySparkProcess.waitFor(); if (exitCode != 0) { logger.info("Finished execution of PySpark job [FAILURE] [Status code: {}]", new Object[] { exitCode }); session.transfer(flowFile, REL_FAILURE); } else { logger.info("Finished execution of PySpark job [SUCCESS] [Status code: {}]", new Object[] { exitCode }); session.transfer(flowFile, REL_SUCCESS); } } catch (final Exception e) { logger.error("Unable to execute PySpark job [FAILURE]", new Object[] { flowFile, e }); session.transfer(flowFile, REL_FAILURE); } } @Override protected Collection<ValidationResult> customValidate(ValidationContext validationContext) { final List<ValidationResult> results = new ArrayList<>(); final String sparkMaster = validationContext.getProperty(SPARK_MASTER).evaluateAttributeExpressions() .getValue().trim().toLowerCase(); final String sparkYarnDeployMode = validationContext.getProperty(SPARK_YARN_DEPLOY_MODE) .evaluateAttributeExpressions().getValue(); final String pySparkAppArgs = validationContext.getProperty(PYSPARK_APP_ARGS).evaluateAttributeExpressions() .getValue(); final String additionalSparkConfigOptions = validationContext.getProperty(ADDITIONAL_SPARK_CONFIG_OPTIONS) .evaluateAttributeExpressions().getValue(); PySparkUtils pySparkUtils = new PySparkUtils(); if ((!sparkMaster.contains("local")) && (!sparkMaster.equals("yarn")) && (!sparkMaster.contains("mesos")) && (!sparkMaster.contains("spark"))) { results.add(new ValidationResult.Builder().subject(this.getClass().getSimpleName()).valid(false) .explanation( "invalid spark master provided. Valid values will have local, local[n], local[*], yarn, mesos, spark") .build()); } if (sparkMaster.equals("yarn")) { if (!(sparkYarnDeployMode.equals("client") || sparkYarnDeployMode.equals("cluster"))) { results.add(new ValidationResult.Builder().subject(this.getClass().getSimpleName()).valid(false) .explanation( "yarn master requires a deploy mode to be specified as either 'client' or 'cluster'") .build()); } } if (!StringUtils.isEmpty(pySparkAppArgs)) { if (!pySparkUtils.validateCsvArgs(pySparkAppArgs)) { results.add(new ValidationResult.Builder().subject(this.getClass().getSimpleName()).valid(false) .explanation( "app args in invalid format. They should be provided as arg1,arg2,arg3 and so on.") .build()); } } if (!StringUtils.isEmpty(additionalSparkConfigOptions)) { if (!pySparkUtils.validateKeyValueArgs(additionalSparkConfigOptions)) { results.add(new ValidationResult.Builder().subject(this.getClass().getSimpleName()).valid(false) .explanation( "additional spark config options in invalid format. They should be provided as config1=value1,config2=value2 and so on.") .build()); } } return results; } }