List of usage examples for com.amazonaws.services.elasticmapreduce.model DescribeJobFlowsRequest DescribeJobFlowsRequest
public DescribeJobFlowsRequest()
From source file:awswc.AwsConsoleApp.java
License:Open Source License
public static void wasFinished(RunJobFlowResult runJobFlowResult) throws InterruptedException { DescribeJobFlowsRequest describeJobFlowsRequest = new DescribeJobFlowsRequest() .withJobFlowIds(runJobFlowResult.getJobFlowId()); int secondsBetweenPolling = 120; String state = null;// ww w .ja v a 2 s . c o m do { Thread.sleep(secondsBetweenPolling * 1000); DescribeJobFlowsResult jobFlowsResult = emr.describeJobFlows(describeJobFlowsRequest); JobFlowDetail detail = jobFlowsResult.getJobFlows().get(0); JobFlowExecutionStatusDetail executionStatusDetail = detail.getExecutionStatusDetail(); state = executionStatusDetail.getState(); } while (state != null && !state.equals("COMPLETE") && !state.equals("FAILED") && !state.equals("TERMINATED")); }
From source file:com.tstordyallison.ffmpegmr.emr.JobflowConfiguration.java
License:Open Source License
public JobflowConfiguration() { try {/*from w ww . j a v a2 s . c o m*/ Gson gson = new GsonBuilder().disableHtmlEscaping().create(); jobflow = gson.fromJson(new BufferedReader(new FileReader(JOB_FLOW_INFO_FILE)), JobflowConfiguration.JobFlow.class); instance = gson.fromJson(new BufferedReader(new FileReader(INSTANCE_INFO_FILE)), JobflowConfiguration.Instance.class); // We must also go and get an up to date value for the coreInstanceCount using the API. DescribeJobFlowsResult jobFlowDescription = JobController.getEmr() .describeJobFlows(new DescribeJobFlowsRequest().withJobFlowIds(jobflow.jobFlowId)); JobFlowDetail jobFlowDetail = jobFlowDescription.getJobFlows().get(0); Collection<InstanceGroupDetail> instanceGroups = jobFlowDetail.getInstances().getInstanceGroups(); for (InstanceGroupDetail instanceGroup : instanceGroups) { if (instanceGroup.getInstanceRole().equals("CORE")) { jobflow.coreInstanceCount = instanceGroup.getInstanceRunningCount(); break; } } } catch (Exception e) { System.err.println("WARNING: Unable to read instance configuration files - logs will be inaccurate."); } }
From source file:datameer.awstasks.aws.emr.EmrCluster.java
License:Apache License
protected JobFlowDetail getJobFlowDetail(String jobFlowId) { DescribeJobFlowsResult describeJobFlows = _emrWebService .describeJobFlows(new DescribeJobFlowsRequest().withJobFlowIds(jobFlowId)); List<JobFlowDetail> jobFlows = describeJobFlows.getJobFlows(); if (jobFlows.isEmpty()) { throw new IllegalArgumentException("no job flow with id '" + _jobFlowId + "' found"); }//w ww .jav a 2 s .c o m return jobFlows.get(0); }
From source file:datameer.awstasks.aws.emr.EmrCluster.java
License:Apache License
protected List<JobFlowDetail> getRunningJobFlowDetailsByName(String name) { DescribeJobFlowsResult describeJobFlows = _emrWebService.describeJobFlows(new DescribeJobFlowsRequest() .withJobFlowStates(JobFlowState.STARTING.name(), JobFlowState.BOOTSTRAPPING.name(), JobFlowState.WAITING.name(), JobFlowState.RUNNING.name())); List<JobFlowDetail> jobFlows = describeJobFlows.getJobFlows(); for (Iterator<JobFlowDetail> iterator = jobFlows.iterator(); iterator.hasNext();) { JobFlowDetail jobFlowDetail = iterator.next(); if (!name.equals(jobFlowDetail.getName())) { iterator.remove();// w w w . j a v a 2 s . c o m } } return jobFlows; }
From source file:fr.ens.biologie.genomique.eoulsan.util.cloud.AWSElasticMapReduceJob.java
License:LGPL
/** * Wait the end of the job//from ww w .j a v a 2 s . co m * @param secondBetweenChecking number of seconds to wait between 2 checks * @return the final state of the job */ public String waitForJob(final int secondBetweenChecking) { if (this.runFlowResult == null) { return null; } final DescribeJobFlowsRequest describeJobFlowsRequest = new DescribeJobFlowsRequest() .withJobFlowIds(this.runFlowResult.getJobFlowId()); String state = null; String lastState = null; int failCount = 0; try { do { Thread.sleep(secondBetweenChecking * 1000); try { final DescribeJobFlowsResult jobFlowsResult = this.elasticMapReduceClient .describeJobFlows(describeJobFlowsRequest); final JobFlowDetail detail = jobFlowsResult.getJobFlows().get(0); final JobFlowExecutionStatusDetail executionStatusDetail = detail.getExecutionStatusDetail(); failCount = 0; state = executionStatusDetail.getState(); } catch (AmazonClientException ace) { failCount++; getLogger().warning("Amazon client exception: " + ace.getMessage()); if (failCount >= MAX_FAIL_COUNT) { throw ace; } } if (lastState == null || !lastState.equals(state)) { getLogger().info("State of the job " + this.runFlowResult.getJobFlowId() + ": " + state); lastState = state; } } while (state != null && !state.equals("COMPLETED") && !state.equals("FAILED") && !state.equals("TERMINATED")); return state; } catch (InterruptedException e) { getLogger().warning("Error while waiting AWS Elastic MapReduce Job: " + e.getMessage()); } return null; }
From source file:org.huahinframework.emanager.amazonaws.elasticmapreduce.ElasticMapReduceManager.java
License:Apache License
/** * @param config//from w w w . java2s. c o m * @throws URISyntaxException */ public void runJob(Config config) throws URISyntaxException { RunJobFlowRequest runJobFlowRequest = null; CreateStepConfigger csc = getCreateStepConfigger(config); if (csc == null) { log.error("Step config create error"); return; } if (jobFlowId == null) { runJobFlowRequest = new RunJobFlowRequest().withName(MAP_REDUCE_NAME) .withBootstrapActions( new BootstrapActionConfig().withName(MEMORY_BOOTSTRAP_NAME).withScriptBootstrapAction( new ScriptBootstrapActionConfig().withPath(MEMORY_BOOTSTRAP_URI)), new BootstrapActionConfig().withName(HADOOP_BOOTSTRAP_NAME).withScriptBootstrapAction( new ScriptBootstrapActionConfig().withPath(HADOOP_BOOTSTRAP_URI) .withArgs("--mapred-key-value", "mapred.task.timeout=3600000")), new BootstrapActionConfig().withName(HUAHIN_BOOTSTRAP_NAME).withScriptBootstrapAction( new ScriptBootstrapActionConfig().withPath(emrProperties.getConfigureS3Path()))) .withInstances(setupJobFlowInstancesConfig()); if (!isEmpty(emrProperties.getLogUri())) { runJobFlowRequest.setLogUri(emrProperties.getLogUri()); } List<StepConfig> stepConfigs = new ArrayList<StepConfig>(); if (emrProperties.isDebug()) { StepConfig enableDebugging = new StepConfig().withName(EMR_DEBUGGIN_NAME) .withActionOnFailure(ACTION_ON_TERMINATE) .withHadoopJarStep(new StepFactory().newEnableDebuggingStep()); stepConfigs.add(enableDebugging); } for (StepConfig sc : csc.createStepConfig(config)) { stepConfigs.add(sc); } runJobFlowRequest.setSteps(stepConfigs); try { RunJobFlowResult result = emr.runJobFlow(runJobFlowRequest); jobFlowId = result.getJobFlowId(); checkDate = new Date(); } catch (Exception e) { e.printStackTrace(); log.error(e); } } else { AddJobFlowStepsRequest addJobFlowStepsRequest = new AddJobFlowStepsRequest().withJobFlowId(jobFlowId) .withSteps(csc.createStepConfig(config)); emr.addJobFlowSteps(addJobFlowStepsRequest); } running = true; try { config.setJobFlowId(jobFlowId); QueueUtils.updateQueue(config); } catch (IOException e) { e.printStackTrace(); log.error(e); } int stepSize = 0; String stepStatus = JobUtils.STEP_STATUS_PENDING; while (stepStatus.equals(JobUtils.STEP_STATUS_PENDING) || stepStatus.equals(JobUtils.STEP_STATUS_RUNNING)) { if (sleep()) { break; } DescribeJobFlowsRequest describeJobFlowsRequest = new DescribeJobFlowsRequest() .withJobFlowIds(jobFlowId); DescribeJobFlowsResult describeJobFlowsResult = emr.describeJobFlows(describeJobFlowsRequest); if (describeJobFlowsResult.getJobFlows().size() != 1) { break; } JobFlowDetail jobFlowDetail = describeJobFlowsResult.getJobFlows().get(0); JobFlowInstancesDetail instancesDetail = jobFlowDetail.getInstances(); masterPublicDnsName = instancesDetail.getMasterPublicDnsName(); if (isEmpty(config.getMasterPublicDnsName())) { try { config.setMasterPublicDnsName(masterPublicDnsName); QueueUtils.updateQueue(config); } catch (IOException e) { e.printStackTrace(); log.error(e); } } stepSize = jobFlowDetail.getSteps().size(); for (StepDetail stepDetail : jobFlowDetail.getSteps()) { if (stepDetail.getStepConfig().getName().equals(config.getName())) { stepStatus = stepDetail.getExecutionStatusDetail().getState(); break; } } } if (config.isDeleteOnExit()) { if (config.getJobType() == Config.JOB_TYPE_STREAMING) { S3Utils.delete(s3, config.getArgMap().get("mapper")); S3Utils.delete(s3, config.getArgMap().get("reducer")); } else { S3Utils.delete(s3, config.getRun()); } } // Add More than 256 Steps to a Job Flow(http://goo.gl/JDtsV) if (stepSize >= 255) { instanceTerminate(); } running = false; if (stepStatus.equals(JobUtils.STEP_STATUS_COMPLETED)) { config.setStatus(Config.JOB_STATUS_COMPLETE); } else if (stepStatus.equals(JobUtils.STEP_STATUS_FAILED)) { config.setStatus(Config.JOB_STATUS_ERROR); } else if (terminated) { config.setStatus(Config.JOB_STATUS_CANCEL); } try { QueueUtils.updateQueue(config); } catch (IOException e) { e.printStackTrace(); log.error(e); } }
From source file:org.huahinframework.emanager.util.JobUtils.java
License:Apache License
public static JSONArray listJobFlow(AmazonElasticMapReduce emr) { List<JSONObject> l = new ArrayList<JSONObject>(); DescribeJobFlowsResult describeJobFlowsResult = emr.describeJobFlows(new DescribeJobFlowsRequest()); for (JobFlowDetail jobFlowDetail : describeJobFlowsResult.getJobFlows()) { JobFlowExecutionStatusDetail executionStatusDetail = jobFlowDetail.getExecutionStatusDetail(); Map<String, String> m = new HashMap<String, String>(); m.put(Response.JOB_FLOW, jobFlowDetail.getJobFlowId()); m.put(Response.STATE, executionStatusDetail.getState()); m.put(Response.CREATION_DATE, executionStatusDetail.getCreationDateTime().toString()); m.put(Response.START_DATE, object2String(executionStatusDetail.getStartDateTime(), true)); m.put(Response.END_DATE, object2String(executionStatusDetail.getEndDateTime(), true)); l.add(new JSONObject(m)); }/*w ww.jav a2 s.c o m*/ return new JSONArray(l); }
From source file:org.huahinframework.emanager.util.JobUtils.java
License:Apache License
public static JSONArray runningsJobFlow(AmazonElasticMapReduce emr) { List<JSONObject> l = new ArrayList<JSONObject>(); DescribeJobFlowsRequest describeJobFlowsRequest = new DescribeJobFlowsRequest() .withJobFlowStates(JOB_FLOW_STATUS_STARTING, JOB_FLOW_STATUS_RUNNING, JOB_FLOW_STATUS_WAITING); DescribeJobFlowsResult describeJobFlowsResult = emr.describeJobFlows(describeJobFlowsRequest); for (JobFlowDetail jobFlowDetail : describeJobFlowsResult.getJobFlows()) { JobFlowExecutionStatusDetail executionStatusDetail = jobFlowDetail.getExecutionStatusDetail(); Map<String, Object> m = new HashMap<String, Object>(); m.put(Response.JOB_FLOW, jobFlowDetail.getJobFlowId()); m.put(Response.STATE, executionStatusDetail.getState()); m.put(Response.CREATION_DATE, executionStatusDetail.getCreationDateTime().toString()); m.put(Response.START_DATE, object2String(executionStatusDetail.getStartDateTime(), true)); m.put(Response.END_DATE, object2String(executionStatusDetail.getEndDateTime(), true)); if (!isEmpty(jobFlowDetail.getSteps())) { List<Object> ll = new ArrayList<Object>(); for (StepDetail sd : jobFlowDetail.getSteps()) { Map<String, Object> mm = new HashMap<String, Object>(); StepConfig sc = sd.getStepConfig(); StepExecutionStatusDetail sesd = sd.getExecutionStatusDetail(); mm.put(Response.NAME, sc.getName()); mm.put(Response.ACTION_ON_FAILURE, sc.getActionOnFailure()); mm.put(Response.STATE, object2String(sesd.getState(), false)); mm.put(Response.CREATION_DATE, object2String(sesd.getCreationDateTime(), true)); mm.put(Response.START_DATE, object2String(sesd.getStartDateTime(), true)); mm.put(Response.END_DATE, object2String(sesd.getEndDateTime(), true)); HadoopJarStepConfig hjsc = sc.getHadoopJarStep(); mm.put(Response.JAR, object2String(hjsc.getJar(), false)); mm.put(Response.MAIN_CLASS, object2String(hjsc.getMainClass(), false)); if (!isEmpty(hjsc.getArgs())) { mm.put(Response.ARGS, hjsc.getArgs()); }/*from w w w.j av a 2 s .c om*/ ll.add(mm); } m.put(Response.STEPS, ll); } l.add(new JSONObject(m)); } return new JSONArray(l); }
From source file:org.huahinframework.emanager.util.JobUtils.java
License:Apache License
public static JSONObject getJobFlow(String jobFlow, AmazonElasticMapReduce emr) { DescribeJobFlowsRequest describeJobFlowsRequest = new DescribeJobFlowsRequest().withJobFlowIds(jobFlow); DescribeJobFlowsResult describeJobFlowsResult = emr.describeJobFlows(describeJobFlowsRequest); if (describeJobFlowsResult.getJobFlows().size() != 1) { return new JSONObject(); }/*from w w w . ja va2s. c om*/ JobFlowDetail jobFlowDetail = describeJobFlowsResult.getJobFlows().get(0); JobFlowExecutionStatusDetail executionStatusDetail = jobFlowDetail.getExecutionStatusDetail(); JobFlowInstancesDetail instancesDetail = jobFlowDetail.getInstances(); Map<String, Object> m = new HashMap<String, Object>(); m.put(Response.JOB_FLOW, jobFlowDetail.getJobFlowId()); m.put(Response.STATE, executionStatusDetail.getState()); m.put(Response.CREATION_DATE, executionStatusDetail.getCreationDateTime().toString()); m.put(Response.START_DATE, object2String(executionStatusDetail.getStartDateTime(), true)); m.put(Response.END_DATE, object2String(executionStatusDetail.getEndDateTime(), true)); m.put(Response.AMI_VERSION, object2String(jobFlowDetail.getAmiVersion(), false)); m.put(Response.NAME, jobFlowDetail.getName()); m.put(Response.LOG_URI, object2String(jobFlowDetail.getLogUri(), false)); if (!isEmpty(jobFlowDetail.getSupportedProducts())) { m.put(Response.SUPPORTED_PRODUCTS, jobFlowDetail.getSupportedProducts()); } m.put(Response.EC2_KEY_NAME, object2String(instancesDetail.getEc2KeyName(), false)); m.put(Response.EC2_SUBNET_ID, object2String(instancesDetail.getEc2SubnetId(), false)); m.put(Response.HADOOP_VERSION, object2String(instancesDetail.getHadoopVersion(), false)); m.put(Response.INSTANCE_COUNT, integer2String(instancesDetail.getInstanceCount())); m.put(Response.KEEP_JOB_FLOW_ALIVE_WHEN_NO_STEPS, object2String(instancesDetail.getKeepJobFlowAliveWhenNoSteps(), true)); m.put(Response.MASTER_INSTANCE_ID, object2String(instancesDetail.getMasterInstanceId(), false)); m.put(Response.MASTER_INSTANCE_TYPE, object2String(instancesDetail.getMasterInstanceType(), false)); m.put(Response.MASTER_PUBLIC_DNS_NAME, object2String(instancesDetail.getMasterPublicDnsName(), false)); m.put(Response.AVAILABILITY_ZONE, object2String(instancesDetail.getPlacement().getAvailabilityZone(), false)); m.put(Response.SLAVE_INSTANCE_TYPE, object2String(instancesDetail.getSlaveInstanceType(), false)); if (!isEmpty(jobFlowDetail.getBootstrapActions())) { List<Object> l = new ArrayList<Object>(); for (BootstrapActionDetail bad : jobFlowDetail.getBootstrapActions()) { Map<String, Object> mm = new HashMap<String, Object>(); BootstrapActionConfig bac = bad.getBootstrapActionConfig(); ScriptBootstrapActionConfig sbac = bac.getScriptBootstrapAction(); mm.put(Response.NAME, object2String(bac.getName(), false)); mm.put(Response.PATH, object2String(sbac.getPath(), false)); if (!isEmpty(sbac.getArgs())) { mm.put(Response.ARGS, sbac.getArgs()); } l.add(mm); } m.put(Response.BOOTSTRAP_ACTIONS, l); } if (!isEmpty(jobFlowDetail.getSteps())) { List<Object> l = new ArrayList<Object>(); for (StepDetail sd : jobFlowDetail.getSteps()) { Map<String, Object> mm = new HashMap<String, Object>(); StepConfig sc = sd.getStepConfig(); StepExecutionStatusDetail sesd = sd.getExecutionStatusDetail(); mm.put(Response.NAME, sc.getName()); mm.put(Response.ACTION_ON_FAILURE, sc.getActionOnFailure()); mm.put(Response.STATE, object2String(sesd.getState(), false)); mm.put(Response.CREATION_DATE, object2String(sesd.getCreationDateTime(), true)); mm.put(Response.START_DATE, object2String(sesd.getStartDateTime(), true)); mm.put(Response.END_DATE, object2String(sesd.getEndDateTime(), true)); HadoopJarStepConfig hjsc = sc.getHadoopJarStep(); mm.put(Response.JAR, object2String(hjsc.getJar(), false)); mm.put(Response.MAIN_CLASS, object2String(hjsc.getMainClass(), false)); if (!isEmpty(hjsc.getArgs())) { mm.put(Response.ARGS, hjsc.getArgs()); } l.add(mm); } m.put(Response.STEPS, l); } return new JSONObject(m); }
From source file:org.pentaho.amazon.emr.job.AmazonElasticMapReduceJobExecutor.java
License:Apache License
public Result execute(Result result, int arg1) throws KettleException { Log4jFileAppender appender = null;// w ww . j a v a2 s . c o m String logFileName = "pdi-" + this.getName(); //$NON-NLS-1$ try { appender = LogWriter.createFileAppender(logFileName, true, false); LogWriter.getInstance().addAppender(appender); log.setLogLevel(parentJob.getLogLevel()); } catch (Exception e) { logError(BaseMessages.getString(PKG, "AmazonElasticMapReduceJobExecutor.FailedToOpenLogFile", //$NON-NLS-1$ logFileName, e.toString())); logError(Const.getStackTracker(e)); } try { // create/connect aws service AmazonElasticMapReduceClient emrClient = new AmazonElasticMapReduceClient(awsCredentials); // pull down jar from vfs FileObject jarFile = KettleVFS.getFileObject(buildFilename(jarUrl)); File tmpFile = File.createTempFile("customEMR", "jar"); tmpFile.deleteOnExit(); FileOutputStream tmpFileOut = new FileOutputStream(tmpFile); IOUtils.copy(jarFile.getContent().getInputStream(), tmpFileOut); URL localJarUrl = tmpFile.toURI().toURL(); // find main class in jar String mainClass = getMainClass(localJarUrl); // create staging bucket AmazonS3 s3Client = new AmazonS3Client(awsCredentials); FileSystemOptions opts = new FileSystemOptions(); DefaultFileSystemConfigBuilder.getInstance().setUserAuthenticator(opts, new StaticUserAuthenticator( null, awsCredentials.getAWSAccessKeyId(), awsCredentials.getAWSSecretKey())); FileObject stagingDirFileObject = KettleVFS.getFileObject(stagingDir, getVariables(), opts); String stagingBucketName = stagingDirFileObject.getName().getBaseName(); if (!s3Client.doesBucketExist(stagingBucketName)) { s3Client.createBucket(stagingBucketName); } // delete old jar if needed try { s3Client.deleteObject(stagingBucketName, jarFile.getName().getBaseName()); } catch (Exception ex) { logError(Const.getStackTracker(ex)); } // put jar in s3 staging bucket s3Client.putObject(new PutObjectRequest(stagingBucketName, jarFile.getName().getBaseName(), tmpFile)); // create non-vfs s3 url to jar String stagingS3JarUrl = "s3://" + stagingBucketName + "/" + jarFile.getName().getBaseName(); String stagingS3BucketUrl = "s3://" + stagingBucketName; RunJobFlowRequest runJobFlowRequest = null; RunJobFlowResult runJobFlowResult = null; if (StringUtil.isEmpty(hadoopJobFlowId)) { // create EMR job flow runJobFlowRequest = createJobFlow(stagingS3BucketUrl, stagingS3JarUrl, mainClass); // start EMR job runJobFlowResult = emrClient.runJobFlow(runJobFlowRequest); } else { List<String> jarStepArgs = new ArrayList<String>(); if (!StringUtil.isEmpty(cmdLineArgs)) { StringTokenizer st = new StringTokenizer(cmdLineArgs, " "); while (st.hasMoreTokens()) { String token = st.nextToken(); logBasic("adding args: " + token); jarStepArgs.add(token); } } HadoopJarStepConfig hadoopJarStep = new HadoopJarStepConfig(); hadoopJarStep.setJar(stagingS3JarUrl); hadoopJarStep.setMainClass(mainClass); hadoopJarStep.setArgs(jarStepArgs); StepConfig stepConfig = new StepConfig(); stepConfig.setName("custom jar: " + jarUrl); stepConfig.setHadoopJarStep(hadoopJarStep); List<StepConfig> steps = new ArrayList<StepConfig>(); steps.add(stepConfig); AddJobFlowStepsRequest addJobFlowStepsRequest = new AddJobFlowStepsRequest(); addJobFlowStepsRequest.setJobFlowId(hadoopJobFlowId); addJobFlowStepsRequest.setSteps(steps); emrClient.addJobFlowSteps(addJobFlowStepsRequest); } String loggingIntervalS = environmentSubstitute(loggingInterval); int logIntv = 60; try { logIntv = Integer.parseInt(loggingIntervalS); } catch (NumberFormatException ex) { logError("Unable to parse logging interval '" + loggingIntervalS + "' - using " + "default of 60"); } // monitor it / blocking / logging if desired if (blocking) { try { if (log.isBasic()) { String executionState = "RUNNING"; List<String> jobFlowIds = new ArrayList<String>(); String id = hadoopJobFlowId; if (StringUtil.isEmpty(hadoopJobFlowId)) { id = runJobFlowResult.getJobFlowId(); jobFlowIds.add(id); } while (isRunning(executionState)) { DescribeJobFlowsRequest describeJobFlowsRequest = new DescribeJobFlowsRequest(); describeJobFlowsRequest.setJobFlowIds(jobFlowIds); DescribeJobFlowsResult describeJobFlowsResult = emrClient .describeJobFlows(describeJobFlowsRequest); boolean found = false; for (JobFlowDetail jobFlowDetail : describeJobFlowsResult.getJobFlows()) { if (jobFlowDetail.getJobFlowId().equals(id)) { executionState = jobFlowDetail.getExecutionStatusDetail().getState(); found = true; } } if (!found) { break; } // logBasic(BaseMessages.getString(PKG, "AmazonElasticMapReduceJobExecutor.RunningPercent", setupPercent, // mapPercent, reducePercent)); logBasic(hadoopJobName + " execution status: " + executionState); try { if (isRunning(executionState)) { Thread.sleep(logIntv * 1000); } } catch (InterruptedException ie) { // Ignore } } if ("FAILED".equalsIgnoreCase(executionState)) { result.setStopped(true); result.setNrErrors(1); result.setResult(false); S3Object outObject = s3Client.getObject(stagingBucketName, id + "/steps/1/stdout"); ByteArrayOutputStream outStream = new ByteArrayOutputStream(); IOUtils.copy(outObject.getObjectContent(), outStream); logError(outStream.toString()); S3Object errorObject = s3Client.getObject(stagingBucketName, id + "/steps/1/stderr"); ByteArrayOutputStream errorStream = new ByteArrayOutputStream(); IOUtils.copy(errorObject.getObjectContent(), errorStream); logError(errorStream.toString()); } } } catch (Exception e) { logError(e.getMessage(), e); } } } catch (Throwable t) { t.printStackTrace(); result.setStopped(true); result.setNrErrors(1); result.setResult(false); logError(t.getMessage(), t); } if (appender != null) { LogWriter.getInstance().removeAppender(appender); appender.close(); ResultFile resultFile = new ResultFile(ResultFile.FILE_TYPE_LOG, appender.getFile(), parentJob.getJobname(), getName()); result.getResultFiles().put(resultFile.getFile().toString(), resultFile); } return result; }