Java tutorial
// Copyright (C) 2011-2012 CRS4. // // This file is part of Seal. // // Seal is free software: you can redistribute it and/or modify it // under the terms of the GNU General Public License as published by the Free // Software Foundation, either version 3 of the License, or (at your option) // any later version. // // Seal is distributed in the hope that it will be useful, but // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License // for more details. // // You should have received a copy of the GNU General Public License along // with Seal. If not, see <http://www.gnu.org/licenses/>. package it.crs4.seal.recab; import it.crs4.seal.common.ClusterUtils; import it.crs4.seal.common.ContextAdapter; import it.crs4.seal.common.FormatNameMap; import it.crs4.seal.common.IMRContext; import it.crs4.seal.common.ReadPair; import it.crs4.seal.common.SealToolRunner; import org.apache.commons.cli.*; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.filecache.DistributedCache; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.ClusterStatus; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; import java.net.URI; import java.net.URISyntaxException; import java.io.IOException; import java.io.FileReader; import java.io.Reader; import java.io.UnsupportedEncodingException; import java.util.Collection; public class RecabTable extends Configured implements Tool { private static final Log LOG = LogFactory.getLog(RecabTable.class); public static final int DEFAULT_RED_TASKS_PER_TRACKER = 3; public static final String ASCII = "US-ASCII"; public static final String TableDelim = ","; public static final byte[] TableDelimBytes; static { try { TableDelimBytes = TableDelim.getBytes(ASCII); } catch (UnsupportedEncodingException e) { throw new RuntimeException(ASCII + " character set not supported!"); } } public static final String LocalVariantsFile = "variants_table.data"; public static final String VariantsFileTypeProperty = "seal.recab.variants-table-type"; public static final String VariantsFileTypeVcf = "vcf"; public static final String VariantsFileTypeRod = "rod"; // Whether to only consider SNP variation locations or to consider all variation types. public static final String SnpsOnlyProperty = "seal.recab.snps-only"; // Default: consider only SNPs public static final boolean SnpsOnlyDefault = false; public static class Map extends Mapper<LongWritable, ReadPair, Text, ObservationCount> { private RecabTableMapper impl; private IMRContext<Text, ObservationCount> contextAdapter; protected VariantReader getVariantFileReader(Configuration conf) throws IOException { // known variation sites Reader in = new FileReader(LocalVariantsFile); VariantReader reader = null; String variantsFileType = conf.get(VariantsFileTypeProperty); if (variantsFileType == null) { LOG.warn("Configuration property " + VariantsFileTypeProperty + " isn't set. Assuming variants file is VCF"); variantsFileType = VariantsFileTypeVcf; } if (variantsFileType.equals(VariantsFileTypeVcf)) { VcfVariantReader vcfReader = new VcfVariantReader(in); vcfReader.setReadSnpsOnly(conf.getBoolean(SnpsOnlyProperty, SnpsOnlyDefault)); reader = vcfReader; } else if (variantsFileType.equals(VariantsFileTypeRod)) { reader = new RodFileVariantReader(in); if (!conf.getBoolean(SnpsOnlyProperty, SnpsOnlyDefault)) throw new RuntimeException( "Sorry. Using all variant types is currently not supported for Rod files. Please let the Seal developers know if this is important to you."); } else throw new IllegalArgumentException("unrecognized variants file type set in " + VariantsFileTypeProperty + " (accepted values are " + VariantsFileTypeVcf + " and " + VariantsFileTypeRod + ")"); return reader; } @Override public void setup(Context context) throws IOException { impl = new RecabTableMapper(); contextAdapter = new ContextAdapter<Text, ObservationCount>(context); Configuration conf = context.getConfiguration(); VariantReader reader = getVariantFileReader(conf); impl.setup(reader, contextAdapter, conf); } @Override public void map(LongWritable pos, ReadPair pair, Context context) throws java.io.IOException, InterruptedException { impl.map(pos, pair, contextAdapter); } } public static class Combiner extends Reducer<Text, ObservationCount, Text, ObservationCount> { private RecabTableCombiner impl; private IMRContext<Text, ObservationCount> contextAdapter; @Override public void setup(Context context) throws IOException { contextAdapter = new ContextAdapter<Text, ObservationCount>(context); impl = new RecabTableCombiner(); impl.setup(context.getConfiguration()); } @Override public void reduce(Text key, Iterable<ObservationCount> values, Context context) throws IOException, InterruptedException { impl.reduce(key, values, contextAdapter); } } public static class Red extends Reducer<Text, ObservationCount, Text, Text> { private RecabTableReducer impl; private IMRContext<Text, Text> contextAdapter; @Override public void setup(Context context) throws IOException { contextAdapter = new ContextAdapter<Text, Text>(context); impl = new RecabTableReducer(); impl.setup(context.getConfiguration()); } @Override public void reduce(Text key, Iterable<ObservationCount> values, Context context) throws IOException, InterruptedException { impl.reduce(key, values, contextAdapter); } } private void distributeVariantsFile(RecabTableOptionParser parser) { Configuration conf = getConf(); DistributedCache.createSymlink(conf); // create symlinks in each task's working directory for the distributed files String distPath; String variantsFileType; if (parser.getVcfFile() != null) { variantsFileType = VariantsFileTypeVcf; distPath = parser.getVcfFile().toString(); conf.set(VariantsFileTypeProperty, VariantsFileTypeVcf); } else if (parser.getRodFile() != null) { variantsFileType = VariantsFileTypeRod; distPath = parser.getRodFile().toString(); conf.set(VariantsFileTypeProperty, VariantsFileTypeRod); } else throw new RuntimeException( "BUG!! RecabTableOptionParser defined with getRodFile and getVcfFile both null!"); distPath += "#" + LocalVariantsFile; try { DistributedCache.addCacheFile(new URI(distPath), conf); } catch (URISyntaxException e) { throw new RuntimeException("Invalid syntax in path to variants file. " + e); } } @Override public int run(String[] args) throws Exception { LOG.info("starting"); RecabTableOptionParser parser = new RecabTableOptionParser(); parser.parse(getConf(), args); LOG.info("Using " + parser.getNReduceTasks() + " reduce tasks"); // must be called before creating the job, since the job // *copies* the Configuration. distributeVariantsFile(parser); // Create a Job using the processed conf Job job = new Job(getConf(), "RecabTable " + parser.getInputPaths().get(0)); job.setJarByClass(RecabTable.class); job.setInputFormatClass(FormatNameMap .getInputFormat(job.getConfiguration().get(RecabTableOptionParser.INPUT_FORMAT_CONF, "sam"))); LOG.info("Using input format " + job.getInputFormatClass().getName()); // input paths for (Path p : parser.getInputPaths()) FileInputFormat.addInputPath(job, p); job.setMapperClass(Map.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(ObservationCount.class); job.setCombinerClass(Combiner.class); job.setReducerClass(Red.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); // output FileOutputFormat.setOutputPath(job, parser.getOutputPath()); // Submit the job, then poll for progress until the job is complete boolean result = job.waitForCompletion(true); if (result) { LOG.info("done"); return 0; } else { LOG.fatal(this.getClass().getName() + " failed!"); return 1; } } public static void main(String[] args) throws Exception { int res = new SealToolRunner().run(new RecabTable(), args); System.exit(res); } }