Java tutorial
/* * The Gemma project * * Copyright (c) 2007 University of British Columbia * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * */ package ubic.gemma.web.controller.expression.experiment; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.StopWatch; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.MediaType; import org.springframework.stereotype.Controller; import org.springframework.util.FileCopyUtils; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.servlet.ModelAndView; import org.springframework.web.servlet.view.RedirectView; import ubic.gemma.core.analysis.service.ExpressionDataFileService; import ubic.gemma.core.job.TaskResult; import ubic.gemma.core.job.executor.webapp.TaskRunningService; import ubic.gemma.core.tasks.AbstractTask; import ubic.gemma.model.common.quantitationtype.QuantitationType; import ubic.gemma.model.expression.experiment.ExpressionExperiment; import ubic.gemma.persistence.service.common.quantitationtype.QuantitationTypeService; import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.util.*; /** * For the download of data files from the browser. We can send the 'raw' data for any one quantitation type, with gene * annotations, OR the 'filtered masked' matrix for the expression experiment. * * @author pavlidis */ @SuppressWarnings("unused") // Called from JS @Controller public class ExpressionExperimentDataFetchController { private static final MetaFileType[] META_FILE_TYPES = { new MetaFileType(1, ".base.metadata", "Sequence analysis summary", ".seq.analysis.sum.txt", false, true), new MetaFileType(2, ".alignment.metadata", "Alignment statistics", ".alignment.statistics.txt", false, true), new MetaFileType(3, "MultiQCReports" + File.separatorChar + "multiqc_report.html", "Multi-QC Report", ".multiqc.report.html", false, false), new MetaFileType(4, "configurations" + File.separatorChar, "Additional pipeline configuration settings", ".pipeline.config.txt", true, false) }; @Autowired private TaskRunningService taskRunningService; @Autowired private ExpressionExperimentService expressionExperimentService; @Autowired private ExpressionDataFileService expressionDataFileService; @Autowired private QuantitationTypeService quantitationTypeService; /** * Regular spring MVC request to fetch a file that already has been generated. It is assumed that the file is in the * DATA_DIR. * * @param response response * @param request request */ @RequestMapping(value = "/getData.html", method = RequestMethod.GET, produces = MediaType.APPLICATION_OCTET_STREAM_VALUE) public void downloadFile(HttpServletRequest request, HttpServletResponse response) throws IOException { this.download(response, ExpressionDataFileService.DATA_DIR + request.getParameter("file"), null); } @RequestMapping(value = "/getMetaData.html", method = RequestMethod.GET, produces = MediaType.APPLICATION_OCTET_STREAM_VALUE) public void downloadMetaFile(HttpServletRequest request, HttpServletResponse response) throws IOException { IllegalArgumentException e = new IllegalArgumentException( "The experiment ID and file type ID parameters must be valid identifiers."); try { String eeId = request.getParameter("eeId"); String typeId = request.getParameter("typeId"); if (StringUtils.isBlank(eeId) || StringUtils.isBlank(typeId)) { throw e; } ExpressionExperiment ee = expressionExperimentService.load(Long.parseLong(eeId)); MetaFileType type = this.getType(Integer.parseInt(typeId)); if (type == null) { throw e; } String dir = ExpressionDataFileService.METADATA_DIR + this.getEEFolderName(ee) + File.separatorChar + type.getFileName(ee); File file = new File(dir); // If this is a directory, check if we can read the most recent file. if (type.isDirectory()) { File fNew = this.getNewestFile(file); if (fNew != null) { dir += fNew.getName(); } } this.download(response, dir, type.getDownloadName(ee)); } catch (NumberFormatException ne) { throw e; } } /** * Scans the metadata directory for any files associated with the given experiment. * * @param eeId the id of the experiment to scan for the metadata for. * @return an array of files available in the metadata directory for the given experiment. */ public MetaFile[] getMetadataFiles(Long eeId) { ExpressionExperiment ee = expressionExperimentService.load(eeId); if (ee == null) { throw new IllegalArgumentException("Experiment with given ID does not exist."); } MetaFile[] metaFiles = new MetaFile[ExpressionExperimentDataFetchController.META_FILE_TYPES.length]; int i = 0; for (MetaFileType type : ExpressionExperimentDataFetchController.META_FILE_TYPES) { // Some files are prefixed with the experiments accession File file = new File(ExpressionDataFileService.METADATA_DIR + this.getEEFolderName(ee) + File.separatorChar + type.getFileName(ee)); // Check if we can read the file if (!file.canRead()) { continue; } // If this is a directory, check if we can read the most recent file. if (type.isDirectory() && this.getNewestFile(file) == null) { continue; } metaFiles[i++] = new MetaFile(type.getId(), type.getDisplayName()); } return metaFiles; } /** * AJAX Method - kicks off a job to start generating (if need be) the text based tab delimited co-expression data * file */ public String getCoExpressionDataFile(Long eeId) { ExpressionExperimentDataFetchCommand tc = new ExpressionExperimentDataFetchCommand(); tc.setExpressionExperimentId(eeId); CoExpressionDataWriterJob job = new CoExpressionDataWriterJob(tc); return taskRunningService.submitLocalTask(job); } /** * AJAX Method - kicks off a job to start generating (if need be) the text based tab delimited experiment design * data file */ public String getDataFile(ExpressionExperimentDataFetchCommand command) { DataWriterJob job = new DataWriterJob(command); return taskRunningService.submitLocalTask(job); } /** * AJAX Method - kicks off a job to start generating (if need be) the text based tab delimited differential * expression data file */ public String getDiffExpressionDataFile(Long analysisId) { ExpressionExperimentDataFetchCommand tc = new ExpressionExperimentDataFetchCommand(); tc.setAnalysisId(analysisId); DiffExpressionDataWriterTask job = new DiffExpressionDataWriterTask(tc); return taskRunningService.submitLocalTask(job); } public File getOutputFile(String filename) { String fullFilePath = ExpressionDataFileService.DATA_DIR + filename; File f = new File(fullFilePath); File parentDir = f.getParentFile(); if (!parentDir.exists()) { //noinspection ResultOfMethodCallIgnored parentDir.mkdirs(); } return f; } public void setQuantitationTypeService(QuantitationTypeService quantitationTypeService) { this.quantitationTypeService = quantitationTypeService; } /** * Forms a folder name where the given experiments metadata will be located (within the {@link ExpressionDataFileService#METADATA_DIR} directory). * * @param ee the experiment to get the folder name for. * @return folder name based on the given experiments properties. Usually this will be the experiments short name, * without any splitting suffixes (e.g. for GSE123.1 the folder name would be GSE123). If the short name is empty for * any reason, the experiments ID will be used. */ private String getEEFolderName(ExpressionExperiment ee) { String sName = ee.getShortName(); if (StringUtils.isBlank(sName)) { return ee.getId().toString(); } return sName.replaceAll("\\.\\d+$", ""); } /** * @param file a directory to scan * @return the file in the directory that was last modified, or null, if such file doesn't exist or is not readable. */ private File getNewestFile(File file) { File[] files = file.listFiles(); if (files != null && files.length > 0) { List<File> fList = Arrays.asList(files); // Sort by last modified, we only want the newest file Collections.sort(fList, new Comparator<File>() { @Override public int compare(File file, File t1) { return Long.compare(file.lastModified(), t1.lastModified()); } }); if (fList.get(0).canRead()) { return fList.get(0); } } return null; } /** * @param response the http response to download to. * @param path the file path to download from * @param downloadName this string will be used as a download name for the downloaded file. If null, the filesystem name * of the file will be used. * @throws IOException if the file in the given path can not be read. */ private void download(HttpServletResponse response, String path, String downloadName) throws IOException { if (StringUtils.isBlank(path)) { throw new IllegalArgumentException("The file name cannot be blank"); } File f = new File(path); if (!f.canRead()) { throw new IOException("Cannot read from " + path); } if (StringUtils.isBlank(downloadName)) { downloadName = f.getName(); } response.setContentLength((int) f.length()); response.addHeader("Content-disposition", "attachment; filename=\"" + downloadName + "\""); FileInputStream in = new FileInputStream(f); FileCopyUtils.copy(in, response.getOutputStream()); response.flushBuffer(); try { in.close(); } catch (IOException ignored) { } } private MetaFileType getType(int id) { for (MetaFileType t : ExpressionExperimentDataFetchController.META_FILE_TYPES) { if (t.getId() == id) return t; } return null; } class CoExpressionDataWriterJob extends AbstractTask<TaskResult, ExpressionExperimentDataFetchCommand> { protected Log log = LogFactory.getLog(this.getClass().getName()); CoExpressionDataWriterJob(ExpressionExperimentDataFetchCommand eeId) { super(eeId); } @Override public TaskResult execute() { StopWatch watch = new StopWatch(); watch.start(); assert this.taskCommand != null; Long eeId = this.taskCommand.getExpressionExperimentId(); ExpressionExperiment ee = expressionExperimentService.load(eeId); if (ee == null) { throw new RuntimeException( "No data available (either due to lack of authorization, or use of an invalid entity identifier)"); } File f = expressionDataFileService.writeOrLocateCoexpressionDataFile(ee, false); watch.stop(); log.debug("Finished getting co-expression file; done in " + watch.getTime() + " milliseconds"); ModelAndView mav = new ModelAndView(new RedirectView("/getData.html?file=" + f.getName(), true)); return new TaskResult(taskCommand, mav); } } /** * @author keshav */ class DataWriterJob extends AbstractTask<TaskResult, ExpressionExperimentDataFetchCommand> { protected Log log = LogFactory.getLog(this.getClass().getName()); DataWriterJob(ExpressionExperimentDataFetchCommand command) { super(command); } @Override public TaskResult execute() { StopWatch watch = new StopWatch(); watch.start(); /* 'do yer thang' */ assert this.taskCommand != null; Long qtId = taskCommand.getQuantitationTypeId(); Long eeId = taskCommand.getExpressionExperimentId(); Long eedId = taskCommand.getExperimentalDesignId(); String format = taskCommand.getFormat(); boolean filtered = taskCommand.isFilter(); String usedFormat = "text"; /* make sure a valid/recognizable format is used */ if (StringUtils.isNotBlank(format)) { if (!(format.equals("text") || format.equals("json"))) { throw new RuntimeException("Format " + format + " is not recognized."); } usedFormat = format; } /* if qt is not set, send the data for the filtered matrix */ QuantitationType qType = null; ExpressionExperiment ee; /* design file */ if (eedId != null) { log.debug("Request is for design file."); ee = expressionExperimentService.load(eeId); if (ee == null) { throw new RuntimeException("Expression experiment id " + eeId + " was invalid: doesn't exist in system, or you lack authorization."); } } /* data file */ else { if (qtId != null) { qType = quantitationTypeService.load(qtId); if (qType == null) { throw new RuntimeException("Quantitation type ID " + qtId + " was invalid: doesn't exist in system, or you lack authorization."); } /* paranoia: in case QTs aren't secured properly, make sure we have access to the EE */ ee = expressionExperimentService.findByQuantitationType(qType); } else { ee = expressionExperimentService.load(eeId); } } if (ee == null) { throw new RuntimeException( "No data available (either due to lack of authorization, or use of an invalid entity identifier)"); } ee = expressionExperimentService.thawLite(ee); File f = null; /* write out the file using text format */ if (usedFormat.equals("text")) { /* the design file */ if (eedId != null) { f = expressionDataFileService.writeOrLocateDesignFile(ee, false); } /* the data file */ else { if (qType != null) { log.debug("Using quantitation type to create matrix."); f = expressionDataFileService.writeOrLocateDataFile(qType, false); } else { f = expressionDataFileService.writeOrLocateDataFile(ee, false, filtered); } } } /* json format */ else if (usedFormat.equals("json")) { if (qType != null) { f = expressionDataFileService.writeOrLocateJSONDataFile(qType, false); } else { f = expressionDataFileService.writeOrLocateJSONDataFile(ee, false, filtered); } } if (f == null) throw new IllegalStateException("No file was obtained"); watch.stop(); log.debug("Finished writing and downloading a file; done in " + watch.getTime() + " milliseconds"); ModelAndView mav = new ModelAndView(new RedirectView("/getData.html?file=" + f.getName(), true)); return new TaskResult(taskCommand, mav); } } class DiffExpressionDataWriterTask extends AbstractTask<TaskResult, ExpressionExperimentDataFetchCommand> { protected Log log = LogFactory.getLog(this.getClass().getName()); DiffExpressionDataWriterTask(ExpressionExperimentDataFetchCommand command) { super(command); } @Override public TaskResult execute() { StopWatch watch = new StopWatch(); watch.start(); Collection<File> files = new HashSet<>(); assert this.taskCommand != null; if (this.taskCommand.getAnalysisId() != null) { File f = expressionDataFileService.getDiffExpressionAnalysisArchiveFile(taskCommand.getAnalysisId(), taskCommand.isForceRewrite()); files.add(f); // TODO: Support this case (Do we really use it from somewhere?) // } else if ( this.command.getExpressionExperimentId() != null ) { // // Long eeId = this.command.getExpressionExperimentId(); // ExpressionExperiment ee = expressionExperimentService.load( eeId ); // // if ( ee == null ) { // throw new RuntimeException( // "No data available (either due to lack of authorization, or use of an invalid entity identifier)" ); // } // // files = expressionDataFileService.writeOrLocateDiffExpressionDataFiles( ee, command.isForceRewrite() // ); } else { throw new IllegalArgumentException( "Must provide either experiment or specific analysis to provide"); } watch.stop(); log.debug("Finished writing and downloading differential expression file(s); done in " + watch.getTime() + " milliseconds"); if (files.isEmpty()) { throw new IllegalArgumentException( "No data available (either due to no analyses being present, lack of authorization, or use of an invalid entity identifier)"); // } else if ( files.size() > 1 ) { // throw new UnsupportedOperationException( // "Sorry, you can't get multiple analyses at once using this method." ); } ModelAndView mav = new ModelAndView( new RedirectView("/getData.html?file=" + files.iterator().next().getName(), true)); return new TaskResult(taskCommand, mav); } } } class MetaFileType { private int id; private String fileName; private String displayName; private String downloadName; private Boolean isDirectory; private Boolean isNamePrefixed; /** * @param id the identifier of the meta file type. * @param fileName the name of the file in the filesystem. * @param downloadName the string that will be used as the file name when the user downloads it. This name will always be * prefixed with the accession (sort name) of the experiment. * @param displayName the string that will be displayed publicly to describe this file. * @param isDirectory whether this file represents a directory. * @param isNamePrefixed whether the fileName has to be prefixed with the experiment accession name. */ MetaFileType(int id, String fileName, String displayName, String downloadName, Boolean isDirectory, Boolean isNamePrefixed) { this.id = id; this.displayName = displayName; this.fileName = fileName; this.downloadName = downloadName; this.isDirectory = isDirectory; this.isNamePrefixed = isNamePrefixed; } public int getId() { return id; } String getDisplayName() { return displayName; } String getFileName(ExpressionExperiment ee) { return (isNamePrefixed ? ee.getShortName() : "") + fileName; } Boolean isDirectory() { return isDirectory; } String getDownloadName(ExpressionExperiment ee) { return ee.getShortName() + downloadName; } }