Java tutorial
/* * Copyright 2015 OpenCB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.opencb.opencga.server.ws; import com.wordnik.swagger.annotations.Api; import com.wordnik.swagger.annotations.ApiOperation; import com.wordnik.swagger.annotations.ApiParam; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.glassfish.jersey.media.multipart.FormDataParam; import org.opencb.biodata.models.feature.Region; import org.opencb.datastore.core.*; import org.opencb.opencga.analysis.storage.AnalysisFileIndexer; import org.opencb.opencga.catalog.db.api.CatalogFileDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogIOException; import org.opencb.opencga.analysis.files.FileMetadataReader; import org.opencb.opencga.analysis.files.FileScanner; import org.opencb.opencga.catalog.io.CatalogIOManager; import org.opencb.opencga.catalog.models.DataStore; import org.opencb.opencga.catalog.models.File; import org.opencb.opencga.catalog.models.Index; import org.opencb.opencga.catalog.models.Job; import org.opencb.opencga.catalog.utils.CatalogFileUtils; import org.opencb.opencga.core.common.Config; import org.opencb.opencga.core.common.IOUtils; import org.opencb.opencga.core.common.UriUtils; import org.opencb.opencga.core.exception.VersionException; import org.opencb.opencga.storage.core.StorageManagerException; import org.opencb.opencga.storage.core.alignment.AlignmentStorageManager; import org.opencb.opencga.storage.core.alignment.adaptors.AlignmentDBAdaptor; import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.*; import javax.ws.rs.Path; import javax.ws.rs.QueryParam; import javax.ws.rs.core.*; import java.io.*; import java.net.URI; import java.nio.charset.StandardCharsets; import java.nio.file.*; import java.util.*; @Path("/{version}/files") @Produces(MediaType.APPLICATION_JSON) @Api(value = "Files", position = 4, description = "Methods for working with 'files' endpoint") public class FileWSServer extends OpenCGAWSServer { public FileWSServer(@PathParam("version") String version, @Context UriInfo uriInfo, @Context HttpServletRequest httpServletRequest) throws IOException, ClassNotFoundException, IllegalAccessException, InstantiationException, VersionException { super(version, uriInfo, httpServletRequest); // String alignmentManagerName = properties.getProperty("STORAGE.ALIGNMENT-MANAGER", MONGODB_ALIGNMENT_MANAGER); // String alignmentManagerName = MONGODB_ALIGNMENT_MANAGER; // String variantManagerName = MONGODB_VARIANT_MANAGER; // if (variantStorageManager == null) { // variantStorageManager = (VariantStorageManager) Class.forName(variantManagerName).newInstance(); // } // if(alignmentStorageManager == null) { // alignmentStorageManager = (AlignmentStorageManager) Class.forName(alignmentManagerName).newInstance(); //// try { //// alignmentStorageManager = (AlignmentStorageManager) Class.forName(alignmentManagerName).newInstance(); //// } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { //// e.printStackTrace(); //// logger.error(e.getMessage(), e); //// } // //dbAdaptor = alignmentStorageManager.getDBAdaptor(null); // } } @POST @Path("/create") @Consumes(MediaType.APPLICATION_JSON) @ApiOperation(value = "Create file with POST method", position = 1, response = QueryResult.class, notes = "This method only creates the file entry in Catalog.<br>" + "Will accept (but not yet): acl.<br>" + "<ul>" + "<il><b>id</b> parameter will be ignored.<br></il>" + "<il><b>type</b> accepted values: [<b>'FOLDER', 'FILE', 'INDEX'</b>].<br></il>" + "<il><b>format</b> accepted values: [<b>'PLAIN', 'GZIP', 'EXECUTABLE', 'IMAGE'</b>].<br></il>" + "<il><b>bioformat</b> accepted values: [<b>'VARIANT', 'ALIGNMENT', 'SEQUENCE', 'NONE'</b>].<br></il>" + "<il><b>status</b> accepted values (admin required): [<b>'INDEXING', 'STAGE', 'UPLOADED', 'READY', 'TRASHED', 'DELETED'</b>].<br></il>" + "<il><b>creatorId</b> should be the same as que sessionId user (unless you are admin) </il>" + "<ul>") public Response createFilePOST( @ApiParam(value = "studyId", required = true) @QueryParam("studyId") String studyIdStr, @ApiParam(value = "files", required = true) List<File> files) { // List<File> catalogFiles = new LinkedList<>(); List<QueryResult<File>> queryResults = new LinkedList<>(); int studyId; try { studyId = catalogManager.getStudyId(studyIdStr); } catch (CatalogException e) { e.printStackTrace(); return createErrorResponse(e); } for (File file : files) { try { QueryResult<File> fileQueryResult = catalogManager.createFile(studyId, file.getType(), file.getFormat(), file.getBioformat(), file.getPath(), file.getOwnerId(), file.getCreationDate(), file.getDescription(), file.getStatus(), file.getDiskUsage(), file.getExperimentId(), file.getSampleIds(), file.getJobId(), file.getStats(), file.getAttributes(), true, queryOptions, sessionId); // file = fileQueryResult.getResult().get(0); System.out.println("fileQueryResult = " + fileQueryResult); queryResults.add(fileQueryResult); } catch (Exception e) { queryResults.add(new QueryResult<>("createFile", 0, 0, 0, "", e.getMessage(), Collections.<File>emptyList())); // return createErrorResponse(e.getMessage()); } } return createOkResponse(queryResults); } @GET @Path("/create-folder") @ApiOperation(value = "Create folder", position = 2) public Response createFolder( @ApiParam(value = "studyId", required = true) @QueryParam("studyId") String studyIdStr, @ApiParam(value = "folder", required = true) @QueryParam("folder") String folder) { try { java.nio.file.Path folderPath = Paths.get(folder); boolean parents = true; int studyId = catalogManager.getStudyId(studyIdStr); QueryResult queryResult = catalogManager.createFolder(studyId, folderPath, parents, queryOptions, sessionId); return createOkResponse(queryResult); } catch (Exception e) { return createErrorResponse(e); } } @GET @Path("/{fileId}/info") @ApiOperation(value = "File info", position = 3) public Response info(@PathParam(value = "fileId") @DefaultValue("") @FormDataParam("fileId") String fileId) { try { String[] fieldIdArray = fileId.split(","); List<QueryResult> results = new LinkedList<>(); for (String id : fieldIdArray) { results.add(catalogManager.getFile(catalogManager.getFileId(id), this.queryOptions, sessionId)); } return createOkResponse(results); } catch (Exception e) { return createErrorResponse(e); } } @GET @Path("/{fileId}/uri") @ApiOperation(value = "File uri", position = 3) public Response getUri(@PathParam(value = "fileId") @DefaultValue("") @FormDataParam("fileId") String fileIds) { try { List<QueryResult> results = new LinkedList<>(); for (String fileId : fileIds.split(",")) { QueryResult<File> result = catalogManager.getFile(catalogManager.getFileId(fileId), this.queryOptions, sessionId); URI fileUri = catalogManager.getFileUri(result.first()); results.add(new QueryResult<>(fileId, 0, 1, 1, "", "", Collections.singletonList(fileUri))); } return createOkResponse(results); } catch (Exception e) { return createErrorResponse(e); } } @POST @Path("/upload") @Consumes(MediaType.MULTIPART_FORM_DATA) @ApiOperation(httpMethod = "POST", position = 4, value = "Resource to upload a file by chunks", response = QueryResponse.class) public Response chunkUpload(@FormDataParam("chunk_content") byte[] chunkBytes, @FormDataParam("chunk_content") FormDataContentDisposition contentDisposition, @DefaultValue("") @FormDataParam("chunk_id") String chunk_id, @DefaultValue("false") @FormDataParam("last_chunk") String last_chunk, @DefaultValue("") @FormDataParam("chunk_total") String chunk_total, @DefaultValue("") @FormDataParam("chunk_size") String chunk_size, @DefaultValue("") @FormDataParam("chunk_hash") String chunkHash, @DefaultValue("false") @FormDataParam("resume_upload") String resume_upload, @ApiParam(value = "filename", required = true) @DefaultValue("") @FormDataParam("filename") String filename, @ApiParam(value = "fileFormat", required = true) @DefaultValue("") @FormDataParam("fileFormat") String fileFormat, @ApiParam(value = "bioFormat", required = true) @DefaultValue("") @FormDataParam("bioFormat") String bioFormat, @ApiParam(value = "userId", required = true) @DefaultValue("") @FormDataParam("userId") String userId, // @ApiParam(defaultValue = "projectId", required = true) @DefaultValue("") @FormDataParam("projectId") String projectId, @ApiParam(value = "studyId", required = true) @FormDataParam("studyId") String studyIdStr, @ApiParam(value = "relativeFilePath", required = true) @DefaultValue("") @FormDataParam("relativeFilePath") String relativeFilePath, @ApiParam(value = "description", required = true) @DefaultValue("") @FormDataParam("description") String description, @ApiParam(value = "parents", required = true) @DefaultValue("true") @FormDataParam("parents") boolean parents) { long t = System.currentTimeMillis(); java.nio.file.Path filePath = null; final int studyId; try { studyId = catalogManager.getStudyId(studyIdStr); } catch (Exception e) { return createErrorResponse(e); } try { filePath = Paths.get(catalogManager.getFileUri(studyId, relativeFilePath)); System.out.println(filePath); } catch (CatalogIOException e) { System.out.println("catalogManager.getFilePath"); e.printStackTrace(); } catch (CatalogException e) { e.printStackTrace(); } java.nio.file.Path completedFilePath = filePath.getParent().resolve("_" + filename); java.nio.file.Path folderPath = filePath.getParent().resolve("__" + filename); logger.info(relativeFilePath + ""); logger.info(folderPath + ""); logger.info(filePath + ""); boolean resume = Boolean.parseBoolean(resume_upload); try { logger.info("---resume is: " + resume); if (resume) { logger.info("Resume ms :" + (System.currentTimeMillis() - t)); return createOkResponse(getResumeFileJSON(folderPath)); } int chunkId = Integer.parseInt(chunk_id); int chunkSize = Integer.parseInt(chunk_size); boolean lastChunk = Boolean.parseBoolean(last_chunk); logger.info("---saving chunk: " + chunkId); logger.info("lastChunk: " + lastChunk); // WRITE CHUNK TYPE_FILE if (!Files.exists(folderPath)) { logger.info("createDirectory(): " + folderPath); Files.createDirectory(folderPath); } logger.info("check dir " + Files.exists(folderPath)); // String hash = StringUtils.sha1(new String(chunkBytes)); // logger.info("bytesHash: " + hash); // logger.info("chunkHash: " + chunkHash); // hash = chunkHash; if (chunkBytes.length == chunkSize) { Files.write(folderPath.resolve(chunkId + "_" + chunkBytes.length + "_partial"), chunkBytes); } else { String errorMessage = "Chunk content size (" + chunkBytes.length + ") " + "!= chunk_size (" + chunk_size + ")."; logger.error(errorMessage); return createErrorResponse(new IOException(errorMessage)); } if (lastChunk) { logger.info("lastChunk is true..."); Files.deleteIfExists(completedFilePath); Files.createFile(completedFilePath); List<java.nio.file.Path> chunks = getSortedChunkList(folderPath); logger.info("----ordered chunks length: " + chunks.size()); for (java.nio.file.Path partPath : chunks) { logger.info(partPath.getFileName().toString()); Files.write(completedFilePath, Files.readAllBytes(partPath), StandardOpenOption.APPEND); } IOUtils.deleteDirectory(folderPath); try { QueryResult queryResult = catalogManager.createFile(studyId, File.Format.valueOf(fileFormat.toUpperCase()), File.Bioformat.valueOf(bioFormat.toUpperCase()), relativeFilePath, completedFilePath.toUri(), description, parents, sessionId); return createOkResponse(queryResult); } catch (Exception e) { logger.error(e.toString()); return createErrorResponse(e); } } } catch (IOException e) { System.out.println("e = " + e); // TODO Auto-generated catch block e.printStackTrace(); } logger.info("chunk saved ms :" + (System.currentTimeMillis() - t)); return createOkResponse("ok"); } @GET @Path("/{fileId}/download") @ApiOperation(value = "File download", position = 5) public Response download(@PathParam(value = "fileId") @FormDataParam("fileId") String fileIdStr) { try { DataInputStream stream; int fileId = catalogManager.getFileId(fileIdStr); QueryResult<File> queryResult = catalogManager.getFile(fileId, this.queryOptions, sessionId); File file = queryResult.getResult().get(0); stream = catalogManager.downloadFile(fileId, sessionId); // String content = org.apache.commons.io.IOUtils.toString(stream); return createOkResponse(stream, MediaType.APPLICATION_OCTET_STREAM_TYPE, file.getName()); } catch (Exception e) { return createErrorResponse(e); } } @GET @Path("/{fileId}/content") @ApiOperation(value = "File content", position = 6) public Response content(@PathParam(value = "fileId") @FormDataParam("fileId") String fileIdStr, @ApiParam(value = "start", required = false) @QueryParam("start") @DefaultValue("-1") int start, @ApiParam(value = "limit", required = false) @QueryParam("limit") @DefaultValue("-1") int limit) { try { int fileId = catalogManager.getFileId(fileIdStr); DataInputStream stream = catalogManager.downloadFile(fileId, start, limit, sessionId); // String content = org.apache.commons.io.IOUtils.toString(stream); return createOkResponse(stream, MediaType.TEXT_PLAIN_TYPE); } catch (Exception e) { return createErrorResponse(e); } } @GET @Path("/{fileId}/content-grep") @ApiOperation(value = "File content", position = 7) public Response downloadGrep(@PathParam(value = "fileId") @FormDataParam("fileId") String fileIdStr, @ApiParam(value = "pattern", required = false) @QueryParam("pattern") @DefaultValue(".*") String pattern, @ApiParam(value = "ignoreCase", required = false) @QueryParam("ignoreCase") @DefaultValue("false") Boolean ignoreCase, @ApiParam(value = "multi", required = false) @QueryParam("multi") @DefaultValue("true") Boolean multi) { try { int fileId = catalogManager.getFileId(fileIdStr); DataInputStream stream = catalogManager.grepFile(fileId, pattern, ignoreCase, multi, sessionId); // String content = org.apache.commons.io.IOUtils.toString(stream); return createOkResponse(stream, MediaType.TEXT_PLAIN_TYPE); } catch (Exception e) { return createErrorResponse(e); } } @GET @Path("/content-example") @ApiOperation(value = "File content", position = 8) public Response downloadExample( @ApiParam(value = "toolName", required = true) @DefaultValue("") @QueryParam("toolName") String toolName, @ApiParam(value = "fileName", required = true) @DefaultValue("") @QueryParam("fileName") String fileName) { /** I think this next two lines should be parametrized either in analysis.properties or the manifest.json of each tool **/ String analysisPath = Config.getOpenCGAHome() + "/" + Config.getAnalysisProperties().getProperty("OPENCGA.ANALYSIS.BINARIES.PATH"); String fileExamplesToolPath = analysisPath + "/" + toolName + "/examples/" + fileName; try { InputStream stream = new FileInputStream(fileExamplesToolPath); return createOkResponse(stream, MediaType.APPLICATION_OCTET_STREAM_TYPE, fileName); } catch (Exception e) { return createErrorResponse(e); } } @GET @Path("/download-example") @ApiOperation(value = "File download", position = 9) public Response downloadExampleFile( @ApiParam(value = "toolName", required = true) @DefaultValue("") @QueryParam("toolName") String toolName, @ApiParam(value = "fileName", required = true) @DefaultValue("") @QueryParam("fileName") String fileName) { try { String analysisPath = Config.getGcsaHome() + "/" + Config.getAnalysisProperties().getProperty("OPENCGA.ANALYSIS.BINARIES.PATH"); String fileExamplesToolPath = analysisPath + "/" + toolName + "/examples/" + fileName; InputStream istream = new FileInputStream(fileExamplesToolPath); DataInputStream stream = new DataInputStream(istream); return createOkResponse(stream, MediaType.APPLICATION_OCTET_STREAM_TYPE, fileName); } catch (Exception e) { return createErrorResponse(e); } } @GET @Path("/{fileId}/set-header") @ApiOperation(value = "Set file header", position = 10) public Response setHeader(@PathParam(value = "fileId") @FormDataParam("fileId") int fileId, @ApiParam(value = "header", required = true) @DefaultValue("") @QueryParam("header") String header) { String content = ""; DataInputStream stream; QueryResult<File> fileQueryResult; InputStream streamBody = null; // System.out.println("header: "+header); try { /** Obtain file uri **/ File file = catalogManager.getFile(catalogManager.getFileId(String.valueOf(fileId)), sessionId) .getResult().get(0); URI fileUri = catalogManager.getFileUri(file); System.out.println("getUri: " + fileUri.getPath()); /** Set header **/ stream = catalogManager.downloadFile(fileId, sessionId); content = org.apache.commons.io.IOUtils.toString(stream); String lines[] = content.split(System.getProperty("line.separator")); StringBuilder body = new StringBuilder(); body.append(header); body.append(System.getProperty("line.separator")); for (int i = 0; i < lines.length; i++) { String line = lines[i]; if (!line.startsWith("#")) { body.append(line); if (i != lines.length - 1) body.append(System.getProperty("line.separator")); } } /** Write/Copy file **/ streamBody = new ByteArrayInputStream(body.toString().getBytes(StandardCharsets.UTF_8)); Files.copy(streamBody, Paths.get(fileUri), StandardCopyOption.REPLACE_EXISTING); } catch (Exception e) { return createErrorResponse(e); } // createOkResponse(content, MediaType.TEXT_PLAIN) return createOkResponse(streamBody, MediaType.TEXT_PLAIN_TYPE); } @GET @Path("/{folderId}/files") @ApiOperation(value = "File content", position = 11) public Response getAllFilesInFolder( @PathParam(value = "folderId") @FormDataParam("folderId") String folderIdStr) { QueryResult<File> results; try { int folderId = catalogManager.getFileId(folderIdStr); results = catalogManager.getAllFilesInFolder(folderId, queryOptions, sessionId); } catch (Exception e) { return createErrorResponse(e); } return createOkResponse(results); } @GET @Path("/search") @ApiOperation(value = "File info", position = 12) public Response search(@ApiParam(value = "id", required = false) @DefaultValue("") @QueryParam("id") String id, @ApiParam(value = "studyId", required = true) @DefaultValue("") @QueryParam("studyId") String studyId, @ApiParam(value = "name", required = false) @DefaultValue("") @QueryParam("name") String name, @ApiParam(value = "type", required = false) @DefaultValue("") @QueryParam("type") File.Type type, @ApiParam(value = "path", required = false) @DefaultValue("") @QueryParam("path") String path, @ApiParam(value = "bioformat", required = false) @DefaultValue("") @QueryParam("bioformat") File.Bioformat bioformat, @ApiParam(value = "status", required = false) @DefaultValue("") @QueryParam("status") File.Status status, @ApiParam(value = "directory", required = false) @DefaultValue("") @QueryParam("directory") String directory, @ApiParam(value = "ownerId", required = false) @DefaultValue("") @QueryParam("ownerId") String ownerId, @ApiParam(value = "creationDate", required = false) @DefaultValue("") @QueryParam("creationDate") String creationDate, @ApiParam(value = "modificationDate", required = false) @DefaultValue("") @QueryParam("modificationDate") String modificationDate, @ApiParam(value = "description", required = false) @DefaultValue("") @QueryParam("description") String description, @ApiParam(value = "diskUsage", required = false) @DefaultValue("") @QueryParam("diskUsage") Long diskUsage, @ApiParam(value = "Comma separated sampleIds", required = false) @DefaultValue("") @QueryParam("sampleIds") String sampleIds, @ApiParam(value = "jobId", required = false) @DefaultValue("") @QueryParam("jobId") String jobId, @ApiParam(value = "attributes", required = false) @DefaultValue("") @QueryParam("attributes") String attributes, @ApiParam(value = "numerical attributes", required = false) @DefaultValue("") @QueryParam("nattributes") String nattributes) { try { int studyIdNum = catalogManager.getStudyId(studyId); // TODO this must be changed: only one queryOptions need to be passed QueryOptions query = new QueryOptions(); for (String param : params.keySet()) { try { CatalogFileDBAdaptor.FileFilterOption.valueOf(param.split("\\.")[0]); query.put(param, params.getFirst(param)); } catch (IllegalArgumentException ignore) { } } if (query.containsKey("name") && (query.get("name") == null || query.getString("name").isEmpty())) { query.remove("name"); System.out.println("Name attribute empty, it;s been removed"); } if (!this.queryOptions.containsKey("limit")) { this.queryOptions.put("limit", 1000); System.out.println("Adding a limit of 1000"); } System.out.println("query = " + query.toJson()); QueryResult<File> result = catalogManager.searchFile(studyIdNum, query, this.queryOptions, sessionId); return createOkResponse(result); } catch (Exception e) { return createErrorResponse(e); } } @GET @Path("/{fileId}/list") @ApiOperation(value = "List folder", position = 13) public Response list(@PathParam(value = "fileId") @DefaultValue("") @FormDataParam("fileId") String fileId) { try { int fileIdNum = catalogManager.getFileId(fileId); QueryResult result = catalogManager.getAllFilesInFolder(fileIdNum, this.queryOptions, sessionId); return createOkResponse(result); } catch (Exception e) { return createErrorResponse(e); } } @GET @Path("/{fileId}/index") @ApiOperation(value = "File index", position = 14) public Response index(@ApiParam("fileId") @PathParam(value = "fileId") @DefaultValue("") String fileIdStr, @ApiParam("Output directory id") @DefaultValue("-1") @QueryParam("outdir") String outDirStr, @ApiParam("Annotate variants") @DefaultValue("true") @QueryParam("annotate") boolean annotate) { AnalysisFileIndexer analysisFileIndexer = new AnalysisFileIndexer(catalogManager); try { int outDirId = catalogManager.getFileId(outDirStr); int fileId = catalogManager.getFileId(fileIdStr); if (outDirId < 0) { outDirId = catalogManager.getFileParent(fileId, null, sessionId).first().getId(); } if (!queryOptions.containsKey(AnalysisFileIndexer.PARAMETERS)) { File a = catalogManager.getFile(fileId, sessionId).getResult().get(0); if (a.getBioformat() == File.Bioformat.VARIANT) { queryOptions.put(AnalysisFileIndexer.PARAMETERS, Arrays.asList("--calculate-stats", "--include-stats")); } } QueryResult<Job> queryResult = analysisFileIndexer.index(fileId, outDirId, sessionId, queryOptions); return createOkResponse(queryResult); } catch (Exception e) { return createErrorResponse(e); } } @GET @Path("/{fileId}/fetch") @ApiOperation(value = "File fetch", notes = "DEPRECATED. Use .../files/{fileId}/[variants|alignments] or .../studies/{studyId}/[variants|alignments] instead", position = 15) @Deprecated public Response fetch(@PathParam(value = "fileId") @DefaultValue("") String fileIds, @ApiParam(value = "region", allowMultiple = true, required = true) @DefaultValue("") @QueryParam("region") String region, @ApiParam(value = "view_as_pairs", required = false) @DefaultValue("false") @QueryParam("view_as_pairs") boolean view_as_pairs, @ApiParam(value = "include_coverage", required = false) @DefaultValue("true") @QueryParam("include_coverage") boolean include_coverage, @ApiParam(value = "process_differences", required = false) @DefaultValue("true") @QueryParam("process_differences") boolean process_differences, @ApiParam(value = "histogram", required = false) @DefaultValue("false") @QueryParam("histogram") boolean histogram, @ApiParam(value = "GroupBy: [ct, gene, ensemblGene]", required = false) @DefaultValue("") @QueryParam("groupBy") String groupBy, @ApiParam(value = "variantSource", required = false) @DefaultValue("false") @QueryParam("variantSource") boolean variantSource, @ApiParam(value = "interval", required = false) @DefaultValue("2000") @QueryParam("interval") int interval) { List<Region> regions = new LinkedList<>(); String[] splitFileId = fileIds.split(","); List<Object> results = new LinkedList<>(); for (String r : region.split(",")) { regions.add(new Region(r)); } for (String fileId : splitFileId) { int fileIdNum; File file; URI fileUri; try { fileIdNum = catalogManager.getFileId(fileId); QueryResult<File> queryResult = catalogManager.getFile(fileIdNum, sessionId); file = queryResult.getResult().get(0); fileUri = catalogManager.getFileUri(file); } catch (CatalogException e) { e.printStackTrace(); return createErrorResponse(e); } // if (!file.getType().equals(File.Type.INDEX)) { if (file.getIndex() == null || file.getIndex().getStatus() != Index.Status.READY) { return createErrorResponse("", "File {id:" + file.getId() + " name:'" + file.getName() + "'} " + " is not an indexed file."); } // List<Index> indices = file.getIndices(); // Index index = null; // for (Index i : indices) { // if (i.getStorageEngine().equals(backend)) { // index = i; // } // } ObjectMap indexAttributes = new ObjectMap(file.getIndex().getAttributes()); DataStore dataStore = null; try { dataStore = AnalysisFileIndexer.getDataStore(catalogManager, catalogManager.getStudyIdByFileId(file.getId()), file.getBioformat(), sessionId); } catch (CatalogException e) { e.printStackTrace(); return createErrorResponse(e); } String storageEngine = dataStore.getStorageEngine(); String dbName = dataStore.getDbName(); QueryResult result; switch (file.getBioformat()) { case ALIGNMENT: { //TODO: getChunkSize from file.index.attributes? use to be 200 int chunkSize = indexAttributes.getInt("coverageChunkSize", 200); QueryOptions queryOptions = new QueryOptions(); queryOptions.put(AlignmentDBAdaptor.QO_FILE_ID, Integer.toString(fileIdNum)); queryOptions.put(AlignmentDBAdaptor.QO_BAM_PATH, fileUri.getPath()); //TODO: Make uri-compatible queryOptions.put(AlignmentDBAdaptor.QO_VIEW_AS_PAIRS, view_as_pairs); queryOptions.put(AlignmentDBAdaptor.QO_INCLUDE_COVERAGE, include_coverage); queryOptions.put(AlignmentDBAdaptor.QO_PROCESS_DIFFERENCES, process_differences); queryOptions.put(AlignmentDBAdaptor.QO_INTERVAL_SIZE, interval); queryOptions.put(AlignmentDBAdaptor.QO_HISTOGRAM, histogram); queryOptions.put(AlignmentDBAdaptor.QO_COVERAGE_CHUNK_SIZE, chunkSize); if (indexAttributes.containsKey("baiFileId")) { File baiFile = null; try { baiFile = catalogManager.getFile(indexAttributes.getInt("baiFileId"), sessionId).getResult() .get(0); URI baiUri = catalogManager.getFileUri(baiFile); queryOptions.put(AlignmentDBAdaptor.QO_BAI_PATH, baiUri.getPath()); //TODO: Make uri-compatible } catch (CatalogException e) { e.printStackTrace(); logger.error("Can't obtain bai file for file " + fileIdNum, e); } } AlignmentDBAdaptor dbAdaptor; try { AlignmentStorageManager alignmentStorageManager = storageManagerFactory .getAlignmentStorageManager(storageEngine); dbAdaptor = alignmentStorageManager.getDBAdaptor(dbName); } catch (ClassNotFoundException | IllegalAccessException | InstantiationException | StorageManagerException e) { return createErrorResponse(e); } QueryResult alignmentsByRegion; if (histogram) { if (regions.size() != 1) { return createErrorResponse("", "Histogram fetch only accepts one region."); } alignmentsByRegion = dbAdaptor.getAllIntervalFrequencies(regions.get(0), queryOptions); } else { alignmentsByRegion = dbAdaptor.getAllAlignmentsByRegion(regions, queryOptions); } result = alignmentsByRegion; break; } case VARIANT: { String warningMsg = null; Query query = VariantFetcher.getVariantQuery(queryOptions); query.put(VariantDBAdaptor.VariantQueryParams.REGION.key(), region); // for (Map.Entry<String, List<String>> entry : params.entrySet()) { // List<String> values = entry.getValue(); // String csv = values.get(0); // for (int i = 1; i < values.size(); i++) { // csv += "," + values.get(i); // } // queryOptions.add(entry.getKey(), csv); // } // queryOptions.put("files", Arrays.asList(Integer.toString(fileIdNum))); query.put(VariantDBAdaptor.VariantQueryParams.FILES.key(), fileIdNum); if (params.containsKey("fileId")) { warningMsg = "Do not use param \"fileI\". Use \"" + VariantDBAdaptor.VariantQueryParams.RETURNED_FILES.key() + "\" instead"; if (params.get("fileId").get(0).isEmpty()) { query.put(VariantDBAdaptor.VariantQueryParams.RETURNED_FILES.key(), fileId); } else { List<String> files = params.get("fileId"); query.put(VariantDBAdaptor.VariantQueryParams.RETURNED_FILES.key(), files); } } VariantDBAdaptor dbAdaptor; try { dbAdaptor = storageManagerFactory.getVariantStorageManager(storageEngine).getDBAdaptor(dbName); // dbAdaptor = new CatalogVariantDBAdaptor(catalogManager, dbAdaptor); } catch (ClassNotFoundException | IllegalAccessException | InstantiationException | StorageManagerException e) { return createErrorResponse(e); } QueryResult queryResult; if (histogram) { queryOptions.put("interval", interval); queryResult = dbAdaptor.get(query, queryOptions); // } else if (variantSource) { // queryOptions.put("fileId", Integer.toString(fileIdNum)); // queryResult = dbAdaptor.getVariantSourceDBAdaptor().getAllSources(queryOptions); } else if (!groupBy.isEmpty()) { queryResult = dbAdaptor.groupBy(query, groupBy, queryOptions); } else { //With merge = true, will return only one result. // queryOptions.put("merge", true); // queryResult = dbAdaptor.getAllVariantsByRegionList(regions, queryOptions).get(0); queryResult = dbAdaptor.get(query, queryOptions); } result = queryResult; if (warningMsg != null) { result.setWarningMsg( result.getWarningMsg() == null ? warningMsg : (result.getWarningMsg() + warningMsg)); } break; } default: return createErrorResponse("", "Unknown bioformat '" + file.getBioformat() + '\''); } result.setId(Integer.toString(fileIdNum)); System.out.println("result = " + result); results.add(result); } System.out.println("results = " + results); return createOkResponse(results); } @GET @Path("/{fileId}/variants") @ApiOperation(value = "Fetch variants from a VCF/gVCF file", position = 15) public Response getVariants(@ApiParam(value = "", required = true) @PathParam("fileId") String fileIdCsv, @ApiParam(value = "CSV list of variant ids") @QueryParam("ids") String ids, @ApiParam(value = "CSV list of regions: {chr}:{start}-{end}") @QueryParam("region") String region, @ApiParam(value = "CSV list of chromosomes") @QueryParam("chromosome") String chromosome, @ApiParam(value = "CSV list of genes") @QueryParam("gene") String gene, @ApiParam(value = "Variant type: [SNV, MNV, INDEL, SV, CNV]") @QueryParam("type") String type, @ApiParam(value = "Filter by reference") @QueryParam("reference") String reference, @ApiParam(value = "Filter by alternate") @QueryParam("alternate") String alternate, // @ApiParam(value = "") @QueryParam("studies") String studies, @ApiParam(value = "CSV list of studies to be returned") @QueryParam("returnedStudies") String returnedStudies, @ApiParam(value = "CSV list of samples to be returned") @QueryParam("returnedSamples") String returnedSamples, @ApiParam(value = "CSV list of files to be returned.") @QueryParam("returnedFiles") String returnedFiles, @ApiParam(value = "Variants in specific files") @QueryParam("files") String files, @ApiParam(value = "Minor Allele Frequency: [<|>|<=|>=]{number}") @QueryParam("maf") String maf, @ApiParam(value = "Minor Genotype Frequency: [<|>|<=|>=]{number}") @QueryParam("mgf") String mgf, @ApiParam(value = "Number of missing alleles: [<|>|<=|>=]{number}") @QueryParam("missingAlleles") String missingAlleles, @ApiParam(value = "Number of missing genotypes: [<|>|<=|>=]{number}") @QueryParam("missingGenotypes") String missingGenotypes, @ApiParam(value = "Specify if the variant annotation must exists.") @QueryParam("annotationExists") boolean annotationExists, @ApiParam(value = "Samples with a specific genotype: {samp_1}:{gt_1}(,{gt_n})*(;{samp_n}:{gt_1}(,{gt_n})*)* e.g. HG0097:0/0;HG0098:0/1,1/1") @QueryParam("genotype") String genotype, @ApiParam(value = "Consequence type SO term list. e.g. SO:0000045,SO:0000046") @QueryParam("annot-ct") String annot_ct, @ApiParam(value = "XRef") @QueryParam("annot-xref") String annot_xref, @ApiParam(value = "Biotype") @QueryParam("annot-biotype") String annot_biotype, @ApiParam(value = "Polyphen value: [<|>|<=|>=]{number}") @QueryParam("polyphen") String polyphen, @ApiParam(value = "Sift value: [<|>|<=|>=]{number}") @QueryParam("sift") String sift, // @ApiParam(value = "") @QueryParam("protein_substitution") String protein_substitution, @ApiParam(value = "Conservation score: {conservation_score}[<|>|<=|>=]{number}") @QueryParam("conservation") String conservation, @ApiParam(value = "Alternate Population Frequency: {study}:{population}[<|>|<=|>=]{number}") @QueryParam("alternate_frequency") String alternate_frequency, @ApiParam(value = "Reference Population Frequency: {study}:{population}[<|>|<=|>=]{number}") @QueryParam("reference_frequency") String reference_frequency, @ApiParam(value = "Returned genotype for unknown genotypes. Common values: [0/0, 0|0, ./.]") @QueryParam("unknownGenotype") String unknownGenotype, @ApiParam(value = "Limit the number of returned variants.") @QueryParam("limit") int limit, @ApiParam(value = "Skip some number of variants.") @QueryParam("skip") int skip, @ApiParam(value = "Group variants by: [ct, gene, ensemblGene]", required = false) @DefaultValue("") @QueryParam("groupBy") String groupBy, @ApiParam(value = "Count results", required = false) @QueryParam("count") boolean count, @ApiParam(value = "Calculate histogram. Requires one region.", required = false) @DefaultValue("false") @QueryParam("histogram") boolean histogram, @ApiParam(value = "Histogram interval size", required = false) @DefaultValue("2000") @QueryParam("interval") int interval, @ApiParam(value = "Merge results", required = false) @DefaultValue("false") @QueryParam("merge") boolean merge) { List<QueryResult> results = new LinkedList<>(); try { VariantFetcher variantFetcher = new VariantFetcher(this); String[] splitFileId = fileIdCsv.split(","); for (String fileId : splitFileId) { QueryResult result; result = variantFetcher.variantsFile(region, histogram, groupBy, interval, fileId); results.add(result); } } catch (Exception e) { return createErrorResponse(e); } return createOkResponse(results); } @GET @Path("/{fileId}/alignments") @ApiOperation(value = "Fetch alignments from a BAM file", position = 15) public Response getAlignments(@ApiParam(value = "fileId", required = true) @PathParam("fileId") String fileId) { return createOkResponse("PENDING"); } private ObjectMap getResumeFileJSON(java.nio.file.Path folderPath) throws IOException { ObjectMap objectMap = new ObjectMap(); if (Files.exists(folderPath)) { DirectoryStream<java.nio.file.Path> folderStream = Files.newDirectoryStream(folderPath, "*_partial"); for (java.nio.file.Path partPath : folderStream) { String[] nameSplit = partPath.getFileName().toString().split("_"); ObjectMap chunkInfo = new ObjectMap(); chunkInfo.put("size", Integer.parseInt(nameSplit[1])); objectMap.put(nameSplit[0], chunkInfo); } } return objectMap; } private List<java.nio.file.Path> getSortedChunkList(java.nio.file.Path folderPath) throws IOException { List<java.nio.file.Path> files = new ArrayList<>(); DirectoryStream<java.nio.file.Path> stream = Files.newDirectoryStream(folderPath, "*_partial"); for (java.nio.file.Path p : stream) { logger.info("adding to ArrayList: " + p.getFileName()); files.add(p); } logger.info("----ordered files length: " + files.size()); Collections.sort(files, new Comparator<java.nio.file.Path>() { public int compare(java.nio.file.Path o1, java.nio.file.Path o2) { int id_o1 = Integer.parseInt(o1.getFileName().toString().split("_")[0]); int id_o2 = Integer.parseInt(o2.getFileName().toString().split("_")[0]); return id_o1 - id_o2; } }); return files; } @GET @Path("/{fileId}/update") @ApiOperation(value = "Modify file", position = 16) public Response update(@PathParam(value = "fileId") String fileIdStr) { try { ObjectMap parameters = new ObjectMap(); for (String param : params.keySet()) { if (param.equalsIgnoreCase("sid")) continue; String value = params.get(param).get(0); parameters.put(param, value); } int fileId = catalogManager.getFileId(fileIdStr); QueryResult queryResult = catalogManager.modifyFile(fileId, parameters, sessionId); return createOkResponse(queryResult); } catch (Exception e) { return createErrorResponse(e); } } public static class UpdateFile { // public String name; public File.Format format; public File.Bioformat bioformat; // public String path; public String ownerId; public String creationDate; public String modificationDate; public String description; public Long diskUsage; // public int experimentId; public List<Integer> sampleIds; public Integer jobId; public Map<String, Object> stats; public Map<String, Object> attributes; } @POST @Path("/{fileId}/update") @ApiOperation(value = "Modify file", position = 16) public Response updatePOST(@PathParam(value = "fileId") String fileIdStr, @ApiParam(name = "params", value = "Parameters to modify", required = true) UpdateFile params) { try { int fileId = catalogManager.getFileId(fileIdStr); QueryResult queryResult = catalogManager.modifyFile(fileId, new ObjectMap(jsonObjectMapper.writeValueAsString(params)), sessionId); return createOkResponse(queryResult); } catch (Exception e) { return createErrorResponse(e); } } @GET @Path("/link") @ApiOperation(value = "Link an external file into catalog.", position = 17) public Response link(@ApiParam(required = true) @QueryParam("uri") String uriStr, @ApiParam(required = true) @QueryParam("studyId") String studyIdStr, @ApiParam(required = true) @QueryParam("path") String path, @ApiParam(required = false) @DefaultValue("") @QueryParam("description") String description, @ApiParam(required = false) @DefaultValue("false") @QueryParam("parents") boolean parents, @ApiParam(required = false) @DefaultValue("false") @QueryParam("calculateChecksum") boolean calculateChecksum) { try { URI uri = UriUtils.createUri(uriStr); File file; CatalogFileUtils catalogFileUtils = new CatalogFileUtils(catalogManager); int studyId = catalogManager.getStudyId(studyIdStr); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(uri); if (!ioManager.exists(uri)) { throw new CatalogIOException("File " + uri + " does not exist"); } if (ioManager.isDirectory(uri)) { uri = UriUtils.createDirectoryUri(uriStr); file = catalogFileUtils.linkFolder(studyId, path, parents, description, calculateChecksum, uri, false, false, sessionId); new FileScanner(catalogManager).scan(file, null, FileScanner.FileScannerPolicy.REPLACE, calculateChecksum, false, sessionId); } else { final String filePath; if (path.endsWith("/")) { filePath = path + Paths.get(uri.getPath()).getFileName().toString(); } else { int folders = catalogManager .getAllFiles(studyId, new QueryOptions( CatalogFileDBAdaptor.FileFilterOption.path.toString(), path + "/"), sessionId) .getNumResults(); if (folders != 0) { filePath = path + "/" + Paths.get(uri.getPath()).getFileName().toString(); } else { filePath = path; } } file = catalogManager.createFile(studyId, null, null, filePath, description, parents, -1, sessionId) .first(); file = catalogFileUtils.link(file, calculateChecksum, uri, false, false, sessionId); file = FileMetadataReader.get(catalogManager).setMetadataInformation(file, null, queryOptions, sessionId, false); } return createOkResponse( new QueryResult<>("link", 0, 1, 1, null, null, Collections.singletonList(file))); } catch (Exception e) { return createErrorResponse(e); } } @GET @Path("/{fileId}/relink") @ApiOperation(value = "Change file location. Provided file must be either STAGE or be an external file.", position = 17) public Response relink(@ApiParam(value = "File ID") @PathParam("fileId") @DefaultValue("") String fileIdStr, @ApiParam(value = "new URI", required = true) @QueryParam("uri") String uriStr, @ApiParam(value = "Do calculate checksum for new files", required = false) @DefaultValue("false") @QueryParam("calculateChecksum") boolean calculateChecksum) { try { URI uri = UriUtils.createUri(uriStr); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(uri); if (!ioManager.exists(uri)) { throw new CatalogIOException("File " + uri + " does not exist"); } int fileId = catalogManager.getFileId(fileIdStr); File file = catalogManager.getFile(fileId, sessionId).first(); new CatalogFileUtils(catalogManager).link(file, calculateChecksum, uri, false, true, sessionId); file = catalogManager.getFile(file.getId(), queryOptions, sessionId).first(); file = FileMetadataReader.get(catalogManager).setMetadataInformation(file, null, queryOptions, sessionId, false); return createOkResponse( new QueryResult<>("relink", 0, 1, 1, null, null, Collections.singletonList(file))); } catch (Exception e) { return createErrorResponse(e); } } @GET @Path("/{fileId}/refresh") @ApiOperation(value = "Refresh metadata from the selected file or folder. Return updated files.", position = 17) public Response refresh(@PathParam(value = "fileId") @DefaultValue("") String fileIdStr) { try { int fileId = catalogManager.getFileId(fileIdStr); File file = catalogManager.getFile(fileId, sessionId).first(); List<File> files; CatalogFileUtils catalogFileUtils = new CatalogFileUtils(catalogManager); FileMetadataReader fileMetadataReader = FileMetadataReader.get(catalogManager); if (file.getType() == File.Type.FILE) { File file1 = catalogFileUtils.checkFile(file, false, sessionId); file1 = fileMetadataReader.setMetadataInformation(file1, null, queryOptions, sessionId, false); if (file == file1) { //If the file is the same, it was not modified. Only return modified files. files = Collections.emptyList(); } else { files = Collections.singletonList(file); } } else { List<File> result = catalogManager.getAllFilesInFolder(file.getId(), null, sessionId).getResult(); files = new ArrayList<>(result.size()); for (File f : result) { File file1 = fileMetadataReader.setMetadataInformation(f, null, queryOptions, sessionId, false); if (f != file1) { //Add only modified files. files.add(file1); } } } return createOkResponse(new QueryResult<>("refresh", 0, files.size(), files.size(), null, null, files)); } catch (Exception e) { return createErrorResponse(e); } } @GET @Path("/{fileId}/delete") @ApiOperation(value = "Delete file", position = 17) public Response deleteGET(@PathParam(value = "fileId") @DefaultValue("") String fileIdStr) { try { int fileId = catalogManager.getFileId(fileIdStr); QueryResult result = catalogManager.deleteFile(fileId, sessionId); return createOkResponse(result); } catch (Exception e) { return createErrorResponse(e); } } }