it.crs4.seal.demux.DemuxReducer.java Source code

Java tutorial

Introduction

Here is the source code for it.crs4.seal.demux.DemuxReducer.java

Source

// Copyright (C) 2011-2012 CRS4.
//
// This file is part of Seal.
//
// Seal is free software: you can redistribute it and/or modify it
// under the terms of the GNU General Public License as published by the Free
// Software Foundation, either version 3 of the License, or (at your option)
// any later version.
//
// Seal is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
// or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
// for more details.
//
// You should have received a copy of the GNU General Public License along
// with Seal.  If not, see <http://www.gnu.org/licenses/>.

package it.crs4.seal.demux;

import it.crs4.seal.common.IMRContext;
import it.crs4.seal.common.SequenceId;
import it.crs4.seal.common.Utils;

import org.seqdoop.hadoop_bam.SequencedFragment;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;

import java.io.IOException;
import java.util.Iterator;

public class DemuxReducer {
    private static final Log LOG = LogFactory.getLog(DemuxReducer.class);
    private static final byte[] SLASH_X = "/X".getBytes();

    private BarcodeLookup barcodeLookup;
    private Text outputKey = new Text();
    private boolean expectIndexRead = true;
    private boolean separatesReads = false;

    public void setup(String localSampleSheetPath, Configuration conf) throws IOException {
        // load the sample sheet
        Path path = new Path(localSampleSheetPath).makeQualified(FileSystem.getLocal(conf));
        SampleSheet sampleSheet;
        try {
            sampleSheet = DemuxUtils.loadSampleSheet(path, conf);
        } catch (SampleSheet.FormatException e) {
            throw new RuntimeException("Error loading sample sheet.  Message: " + e.getMessage());
        }
        barcodeLookup = new BarcodeLookup(sampleSheet,
                conf.getInt(Demux.CONF_MAX_MISMATCHES, Demux.DEFAULT_MAX_MISMATCHES));

        expectIndexRead = !conf.getBoolean(Demux.CONF_NO_INDEX_READS, false);
        separatesReads = conf.getBoolean(Demux.CONF_SEPARATE_READS, false);
    }

    public void reduce(SequenceId key, Iterable<SequencedFragment> sequences,
            IMRContext<Text, SequencedFragment> context) throws IOException, InterruptedException {
        // XXX: this function is growing too much.  Consider refactoring.
        //////////////////////////////////////////
        // Fragments should all have non-null Read and Lane, as verified by the Mapper.
        // They should be ordered read 2, read 1, read 3 and over
        //////////////////////////////////////////
        Iterator<SequencedFragment> seqs_it = sequences.iterator();
        SequencedFragment fragment;
        String flowcellId = "";
        String indexSeq = ""; // default index is blank
        String sampleId;
        String project;

        fragment = seqs_it.next();

        if (expectIndexRead) {
            // Fetch the first fragment from the list -- it should be the index sequence
            if (fragment.getRead() != 2)
                throw new RuntimeException("Missing read 2 in multiplexed input for location " + key.getLocation()
                        + ".  Record: " + fragment);

            indexSeq = fragment.getSequence().toString();
            // Sequenced tags have an additional 'A' base that separates them from the read. For this
            // reason, when we verify their length we check for BAR_CODE_{MIN,MAX}_LENGTH + 1
            if (indexSeq.length() < (SampleSheet.BAR_CODE_MIN_LENGTH + 1)
                    || indexSeq.length() > (SampleSheet.BAR_CODE_MAX_LENGTH + 1)) {
                throw new RuntimeException(
                        String.format("Unexpected barcode sequence of length %d (expected in interval [%d, %d]",
                                indexSeq.length(), (SampleSheet.BAR_CODE_MIN_LENGTH + 1),
                                (SampleSheet.BAR_CODE_MAX_LENGTH + 1) + "])"));
            }

            // We've consumed this index read.  Advance to the next one.
            fragment = seqs_it.next();
        }

        // From here on, they should be all data reads.

        int lane = fragment.getLane();
        BarcodeLookup.Match m = barcodeLookup.getSampleId(lane, indexSeq);
        if (m == null) {
            sampleId = "unknown";
            project = ".";
        } else {
            sampleId = m.getEntry().getSampleId();
            flowcellId = m.getEntry().getFlowcellId();
            project = m.getEntry().getProject();
            if (project == null)
                project = Demux.DEFAULT_PROJECT;
            context.increment("Barcode base mismatches", String.valueOf(m.getMismatches()), 1);
        }

        // Project/sample results in that directory structure. The key is the same for all reads in iterator
        // TODO:  profile!  We're sanitizing and rebuilding the file name for
        // each set of reads.  It may be a significant waste of CPU that could be fixed by a caching mechanism.
        String keyString = Utils.sanitizeFilename(project) + '/' + Utils.sanitizeFilename(sampleId);
        outputKey.set(keyString);
        if (separatesReads) {
            // append a slash and an 'X' (the latter to make a space for the read number)
            outputKey.append(SLASH_X, 0, SLASH_X.length);
        }

        boolean done = false;
        do {
            fragment.setIndexSequence(indexSeq);

            // When we read qseq, the flowcell id isn't set (the file format doesn't include that data.
            // Since we have the chance here, we'l extract the flowcell id from the sample sheet
            // and set it on the outgoing SequencedFragment.
            if (fragment.getFlowcellId() == null)
                fragment.setFlowcellId(flowcellId);

            if (expectIndexRead && fragment.getRead() > 2)
                fragment.setRead(fragment.getRead() - 1);

            if (separatesReads) {
                // Overwrite the last character of the key with the read number.
                // This technique only supports single digit read numbers
                outputKey.getBytes()[outputKey.getLength() - 1] = (byte) (fragment.getRead().byteValue() + '0');
            }

            context.write(outputKey, fragment);
            context.increment("Sample reads", keyString, 1);

            if (seqs_it.hasNext())
                fragment = seqs_it.next();
            else
                done = true;
        } while (!done);

        if (fragment.getRead() > 2) { // although the code above is generic and will handle any number of reads,
                                      // in our current use cases any more than 2 data reads (non-index) indicate
                                      // a problem with the data.
                                      // XXX: if someone removes this check, verify the "separatesReads" section above.
            throw new RuntimeException("Unexpected output read number " + fragment.getRead() + " at location "
                    + key.getLocation()
                    + " (note that if read number may have been decremented by 1 if an index sequence was present).");
        }
    }
}