Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package testjar; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.Mapper; import org.apache.hadoop.mapred.MapReduceBase; import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.util.Shell.ShellCommandExecutor; import java.util.Random; import java.io.IOException; import java.io.DataOutputStream; import java.io.File; /** * It uses for defining a various types of mapperes with child processes. */ public class GenerateTaskChildProcess { private static final Log LOG = LogFactory.getLog(GenerateTaskChildProcess.class); /** * It uses for defining the string appending mapper with * child processes.It's keep appending the string and increases * the memory continuously. */ public static class StrAppendMapper extends MapReduceBase implements Mapper<LongWritable, Text, NullWritable, NullWritable> { private JobConf conf; public void configure(JobConf conf) { this.conf = conf; } public void map(LongWritable key, Text value, OutputCollector<NullWritable, NullWritable> out, Reporter reporter) throws IOException { int counter = 0; while (counter < 30) { try { reporter.progress(); synchronized (this) { this.wait(1000); } } catch (InterruptedException iexp) { iexp.printStackTrace(); LOG.warn("Interrupted while the map was waiting."); } counter++; } try { createChildProcess(conf, "AppendStr"); } catch (Exception exp) { exp.printStackTrace(); LOG.warn("Exception thrown while creating the child processes"); } } public void close() { } } /** * It uses for defining the String display mapper with child processes. */ public static class StrDisplayMapper extends MapReduceBase implements Mapper<LongWritable, Text, NullWritable, NullWritable> { private JobConf conf; public void configure(JobConf conf) { this.conf = conf; } public void map(LongWritable key, Text value, OutputCollector<NullWritable, NullWritable> out, Reporter reporter) throws IOException { int counter = 0; while (counter < 30) { try { reporter.progress(); synchronized (this) { this.wait(1000); } } catch (InterruptedException iexp) { iexp.printStackTrace(); LOG.warn("Interrupted while the map was waiting."); break; } counter++; } try { createChildProcess(conf, "DispStr"); } catch (Exception exp) { exp.printStackTrace(); LOG.warn("Exception thrown while creating the child processes."); } } } /** * It uses for defining a failed mapper with child processes. * */ public static class FailedMapper extends MapReduceBase implements Mapper<LongWritable, Text, NullWritable, NullWritable> { private JobConf conf; public void configure(JobConf conf) { try { createChildProcess(conf, "failedmapper"); } catch (Exception exp) { exp.printStackTrace(); LOG.warn("Exception throw while creating the child processes"); } } public void map(LongWritable key, Text value, OutputCollector<NullWritable, NullWritable> out, Reporter reporter) throws IOException { int counter = 0; while (counter < 30) { try { reporter.progress(); synchronized (this) { this.wait(1000); } } catch (InterruptedException iexp) { iexp.printStackTrace(); LOG.warn("Interrupted while the map was waiting."); break; } counter++; } throw new RuntimeException("Mapper failed."); } } /** * It uses for failing the map tasks. * */ public static class FailMapper extends MapReduceBase implements Mapper<LongWritable, Text, NullWritable, NullWritable> { public void map(LongWritable key, Text value, OutputCollector<NullWritable, NullWritable> out, Reporter reporter) throws IOException { throw new RuntimeException("failing the map"); } } /** * It uses for creating the child processes for a task. * @param conf configuration for a job. * @param jobName the name of the mapper job. * @throws IOException if an I/O error occurs. */ private static void createChildProcess(JobConf conf, String jobName) throws IOException { FileSystem fs = FileSystem.getLocal(conf); File TMP_ROOT_DIR = new File("/tmp"); String TEST_ROOT_DIR = TMP_ROOT_DIR.getAbsolutePath() + Path.SEPARATOR + "ChildProc_" + jobName; Path scriptDir = new Path(TEST_ROOT_DIR); int numOfChildProcesses = 2; if (fs.exists(scriptDir)) { fs.delete(scriptDir, true); } fs.mkdirs(scriptDir); fs.setPermission(scriptDir, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); String scriptDirName = scriptDir.toUri().getPath(); Random rm = new Random(); String scriptName = "ShellScript_" + jobName + "_" + rm.nextInt() + ".sh"; Path scriptPath = new Path(scriptDirName, scriptName); String shellScript = scriptPath.toString(); String script = null; if (jobName.equals("AppendStr")) { script = "#!/bin/sh\n" + "umask 000\n" + "StrVal=\"Hadoop is framework for data intensive " + "distributed applications.\"\n" + "StrVal=\"${StrVal}Hadoop enables applications to work " + "with thousands of nodes.\"\n" + "echo $StrVal\n" + "if [ \"X$1\" != \"X0\" ]\nthen\n" + " sh " + shellScript + " $(($1-1))\n" + "else\n" + " while(true)\n" + " do\n" + " StrVal=\"$StrVal Hadoop \"\n" + " done\n" + "fi"; } else if (jobName.equals("DispStr")) { script = "#!/bin/sh\n" + "umask 000\n" + "msg=Welcome\n" + "echo $msg\n" + " if [ \"X$1\" != \"X0\" ]\nthen\n" + " sh " + shellScript + " $(($1-1))\n" + "else\n" + " while(true)\n" + " do\n" + " sleep 2 \n" + " done\n" + "fi"; } else { script = "#!/bin/sh\n" + "umask 000\n" + "msg=Welcome\n" + "echo $msg\n" + " if [ \"X$1\" != \"X0\" ]\nthen\n" + " sh " + shellScript + " $(($1-1))\n" + "else\n" + " for count in {1..1000}\n" + " do\n" + " echo \"$msg_$count\" \n" + " done\n" + "fi"; } DataOutputStream file = fs.create(scriptPath); file.writeBytes(script); file.close(); File scriptFile = new File(scriptDirName, scriptName); scriptFile.setExecutable(true); LOG.info("script absolute path:" + scriptFile.getAbsolutePath()); String[] cmd = new String[] { scriptFile.getAbsolutePath(), String.valueOf(numOfChildProcesses) }; ShellCommandExecutor shellExec = new ShellCommandExecutor(cmd); shellExec.execute(); } }