List of usage examples for org.apache.commons.lang3 StringUtils isNumeric
public static boolean isNumeric(final CharSequence cs)
Checks if the CharSequence contains only Unicode digits.
From source file:org.omnaest.utils.operation.foreach.Range.java
/** * Allows to specify a {@link Range} with a given {@link String} expression.<br> * <br>//from w ww . j a v a 2 s .c om * The expression format is:<br> * * <pre> * new Range( "1-5" ); //results in 1,2,3,4,5 * new Range( "1-5:2" ); //results in 1,3,5 * new Range( "123" ); //result in only a single value * * </pre> * * @see Range * @param rangeExpression */ public Range(String rangeExpression) { super(); Assert.isNotNull(rangeExpression); ReplacementResult replacementResult = new StringReplacer("\\:([0-9]*)$").setGroup(1) .findAndRemoveFirst(rangeExpression); final boolean hasDeclaredStep = replacementResult.hasMatchingTokens(); if (hasDeclaredStep) { String[] matchingTokens = replacementResult.getMatchingTokens(); if (matchingTokens != null && matchingTokens.length == 1) { final String stepString = matchingTokens[0]; Assert.isTrue(StringUtils.isNumeric(stepString), "Step must be numerical but was " + stepString); this.step = NumberUtils.toLong(stepString, DEFAULT_STEP); } } rangeExpression = replacementResult.getOutput(); String[] tokens = rangeExpression.split("-"); Assert.isTrue(tokens.length == 2 || tokens.length == 1); if (tokens.length == 2) { Assert.isTrue(StringUtils.isNumeric(tokens[0]), "Range start must be numerical but was " + tokens[0]); Assert.isTrue(StringUtils.isNumeric(tokens[1]), "Range start must be numerical but was " + tokens[1]); this.numberFrom = Long.valueOf(tokens[0]); this.numberTo = Long.valueOf(tokens[1]); } else if (tokens.length == 1) { Assert.isTrue(StringUtils.isNumeric(tokens[0]), "Range start and end must be numerical but was " + tokens[0]); this.numberFrom = this.numberTo = Long.valueOf(tokens[0]); } if (!hasDeclaredStep) { this.step = determineDefaultStep(this.numberFrom, this.numberTo); } Assert.isTrue( this.numberTo == this.numberFrom || Math.signum(this.numberTo - this.numberFrom) == Math.signum(this.step), "The given end number cannot be reached by the given start number and step " + this); }
From source file:org.opencb.biodata.models.variant.Variant.java
private Integer getCopyNumberFromStr(String cnvStr) { String copyNumberString = cnvStr.split(CNVSTR)[1].split(">")[0]; if (StringUtils.isNumeric(copyNumberString)) { return Integer.valueOf(copyNumberString); } else {/*from ww w .java 2 s . c om*/ return null; } }
From source file:org.opencb.biodata.models.variant.VariantAggregatedVcfFactory.java
protected void addInfo(Variant variant, StudyEntry file, int numAllele, Map<String, String> info) { for (Map.Entry<String, String> infoElement : info.entrySet()) { String infoTag = infoElement.getKey(); String infoValue = infoElement.getValue(); switch (infoTag) { case "ACC": // Managing accession ID for the allele String[] ids = infoValue.split(COMMA); file.addAttribute(infoTag, ids[numAllele]); break; // next is commented to store the AC, AF and AN as-is, to be able to compute stats from the DB using the attributes, and "ori" tag // case "AC": // String[] counts = infoValue.split(COMMA); // file.addAttribute(infoTag, counts[numAllele]); // break; // case "AF": // String[] frequencies = infoValue.split(COMMA); // file.addAttribute(infoTag, frequencies[numAllele]); // break; // case "AN": // file.addAttribute(infoTag, "2"); // break; case "NS": // Count the number of samples that are associated with the allele file.addAttribute(infoTag, String.valueOf(file.getSamplesData().size())); break; case "DP": int dp = 0; for (String sampleName : file.getSamplesName()) { String sampleDp = file.getSampleData(sampleName, "DP"); if (StringUtils.isNumeric(sampleDp)) { dp += Integer.parseInt(sampleDp); }//from w w w. j ava 2 s .co m } file.addAttribute(infoTag, String.valueOf(dp)); break; case "MQ": case "MQ0": int mq = 0; int mq0 = 0; for (String sampleName : file.getSamplesName()) { if (StringUtils.isNumeric(file.getSampleData(sampleName, "GQ"))) { int gq = Integer.parseInt(file.getSampleData(sampleName, "GQ")); mq += gq * gq; if (gq == 0) { mq0++; } } } file.addAttribute("MQ", String.valueOf(mq)); file.addAttribute("MQ0", String.valueOf(mq0)); break; default: file.addAttribute(infoTag, infoValue); break; } } }
From source file:org.opencb.opencga.app.cli.analysis.VariantCommandExecutor.java
private void stats() throws CatalogException, AnalysisExecutionException, IOException, ClassNotFoundException, StorageManagerException, InstantiationException, IllegalAccessException, URISyntaxException { AnalysisCliOptionsParser.StatsVariantCommandOptions cliOptions = variantCommandOptions.statsVariantCommandOptions; long studyId = catalogManager.getStudyId(cliOptions.studyId, sessionId); VariantStorage variantStorage = new VariantStorage(catalogManager); QueryOptions options = new QueryOptions() .append(VariantStatisticsManager.OUTPUT_FILE_NAME, cliOptions.fileName) // .append(AnalysisFileIndexer.CREATE, cliOptions.create) // .append(AnalysisFileIndexer.LOAD, cliOptions.load) // .append(AnalysisFileIndexer.LOG_LEVEL, cliOptions.commonOptions.logLevel) // unused .append(VariantStorageManager.Options.UPDATE_STATS.key(), cliOptions.updateStats) .append(VariantStorageManager.Options.AGGREGATED_TYPE.key(), cliOptions.aggregated) .append(VariantStorageManager.Options.AGGREGATION_MAPPING_PROPERTIES.key(), cliOptions.aggregationMappingFile); options.putIfNotEmpty(VariantStorageManager.Options.FILE_ID.key(), cliOptions.fileId); options.putAll(cliOptions.commonOptions.params); List<Long> cohortIds = new LinkedList<>(); if (StringUtils.isNotBlank(cliOptions.cohortIds)) { for (String cohort : cliOptions.cohortIds.split(",")) { if (StringUtils.isNumeric(cohort)) { cohortIds.add(Long.parseLong(cohort)); } else { QueryResult<Cohort> result = catalogManager.getAllCohorts(studyId, new Query(CohortDBAdaptor.QueryParams.NAME.key(), cohort), new QueryOptions("include", "projects.studies.cohorts.id"), sessionId); if (result.getResult().isEmpty()) { throw new CatalogException("Cohort \"" + cohort + "\" not found!"); } else { cohortIds.add(result.first().getId()); }/* w ww . j a v a 2 s. c om*/ } } } variantStorage.calculateStats(studyId, cohortIds, cliOptions.catalogPath, cliOptions.outdir, sessionId, options); // QueryResult<Job> result = variantStorage.calculateStats(outDirId, cohortIds, sessionId, options); // if (cliOptions.job.queue) { // System.out.println(new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(result)); // } }
From source file:org.opencb.opencga.app.cli.analysis.VariantCommandExecutor.java
@Deprecated private void stats(Job job) throws ClassNotFoundException, InstantiationException, CatalogException, IllegalAccessException, IOException, StorageManagerException { AnalysisCliOptionsParser.StatsVariantCommandOptions cliOptions = variantCommandOptions.statsVariantCommandOptions; // long inputFileId = catalogManager.getFileId(cliOptions.fileId); // 1) Initialize VariantStorageManager long studyId = catalogManager.getStudyId(cliOptions.studyId, sessionId); Study study = catalogManager.getStudy(studyId, sessionId).first(); /*//w w w . j a v a 2 s . c o m * Getting VariantStorageManager * We need to find out the Storage Engine Id to be used from Catalog */ DataStore dataStore = AbstractFileIndexer.getDataStore(catalogManager, studyId, File.Bioformat.VARIANT, sessionId); initVariantStorageManager(dataStore); /* * Create DBAdaptor */ VariantDBAdaptor dbAdaptor = variantStorageManager.getDBAdaptor(dataStore.getDbName()); StudyConfigurationManager studyConfigurationManager = dbAdaptor.getStudyConfigurationManager(); StudyConfiguration studyConfiguration = studyConfigurationManager.getStudyConfiguration((int) studyId, null) .first(); /* * Parse Options */ ObjectMap options = storageConfiguration.getStorageEngine(variantStorageManager.getStorageEngineId()) .getVariant().getOptions(); options.put(VariantStorageManager.Options.DB_NAME.key(), dataStore.getDbName()); options.put(VariantStorageManager.Options.OVERWRITE_STATS.key(), cliOptions.overwriteStats); options.put(VariantStorageManager.Options.UPDATE_STATS.key(), cliOptions.updateStats); if (cliOptions.region != null) { options.putIfNotNull(VariantDBAdaptor.VariantQueryParams.REGION.key(), cliOptions.region); } long fileId = catalogManager.getFileId(cliOptions.fileId, sessionId); if (fileId != 0) { options.put(VariantStorageManager.Options.FILE_ID.key(), fileId); } options.put(VariantStorageManager.Options.STUDY_ID.key(), studyId); if (cliOptions.commonOptions.params != null) { options.putAll(cliOptions.commonOptions.params); } Map<String, Integer> cohortIds = new HashMap<>(); Map<String, Set<String>> cohorts = new HashMap<>(); Properties aggregationMappingProperties = null; if (isNotEmpty(cliOptions.aggregationMappingFile)) { aggregationMappingProperties = new Properties(); try (InputStream is = new FileInputStream(cliOptions.aggregationMappingFile)) { aggregationMappingProperties.load(is); options.put(VariantStorageManager.Options.AGGREGATION_MAPPING_PROPERTIES.key(), aggregationMappingProperties); } catch (FileNotFoundException e) { logger.error("Aggregation mapping file {} not found. Population stats won't be parsed.", cliOptions.aggregationMappingFile); } } List<String> cohortNames; if (isEmpty(cliOptions.cohortIds)) { if (aggregationMappingProperties == null) { throw new IllegalArgumentException("Missing cohorts"); } else { cohortNames = new LinkedList<>( VariantAggregatedStatsCalculator.getCohorts(aggregationMappingProperties)); } } else { cohortNames = Arrays.asList(cliOptions.cohortIds.split(",")); } for (String cohort : cohortNames) { int cohortId; if (StringUtils.isNumeric(cohort)) { cohortId = Integer.parseInt(cohort); } else { if (studyConfiguration.getCohortIds().containsKey(cohort)) { cohortId = studyConfiguration.getCohortIds().get(cohort); } else { throw new IllegalArgumentException("Unknown cohort name " + cohort); } } Set<String> samples = studyConfiguration.getCohorts().get(cohortId).stream() .map(sampleId -> studyConfiguration.getSampleIds().inverse().get(sampleId)) .collect(Collectors.toSet()); cohorts.put(studyConfiguration.getCohortIds().inverse().get(cohortId), samples); } options.put(VariantStorageManager.Options.AGGREGATED_TYPE.key(), cliOptions.aggregated); /* * Create and load stats */ // URI outputUri = UriUtils.createUri(cliOptions.fileName == null ? "" : cliOptions.fileName); // URI outputUri = job.getTmpOutDirUri(); URI outputUri = IndexDaemon.getJobTemporaryFolder(job.getId(), catalogConfiguration.getTempJobsDir()) .toUri(); String filename; if (isEmpty(cliOptions.fileName)) { filename = VariantStorageManager.buildFilename(studyConfiguration.getStudyName(), (int) fileId); } else { filename = cliOptions.fileName; } // assertDirectoryExists(directoryUri); VariantStatisticsManager variantStatisticsManager = new VariantStatisticsManager(); boolean doCreate = true; boolean doLoad = true; // doCreate = statsVariantsCommandOptions.create; // doLoad = statsVariantsCommandOptions.load != null; // if (!statsVariantsCommandOptions.create && statsVariantsCommandOptions.load == null) { // doCreate = doLoad = true; // } else if (statsVariantsCommandOptions.load != null) { // filename = statsVariantsCommandOptions.load; // } QueryOptions queryOptions = new QueryOptions(options); if (doCreate) { filename += "." + TimeUtils.getTime(); outputUri = outputUri.resolve(filename); outputUri = variantStatisticsManager.createStats(dbAdaptor, outputUri, cohorts, cohortIds, studyConfiguration, queryOptions); } if (doLoad) { outputUri = outputUri.resolve(filename); variantStatisticsManager.loadStats(dbAdaptor, outputUri, studyConfiguration, queryOptions); } }
From source file:org.opencb.opencga.app.cli.main.OpenCGAMainOld.java
private int runCommand(OptionsParser optionsParser) throws Exception { int returnValue = 0; if (catalogManager == null && !optionsParser.getSubCommand().isEmpty()) { CatalogConfiguration catalogConfiguration = CatalogConfiguration.load(new FileInputStream( Paths.get(Config.getOpenCGAHome(), "conf", "catalog-configuration.yml").toFile())); catalogManager = new CatalogManager(catalogConfiguration); }/*ww w . jav a2s . com*/ String sessionId = login(optionsParser.getUserAndPasswordOptions()); switch (optionsParser.getCommand()) { case "users": switch (optionsParser.getSubCommand()) { case "create": { OptionsParser.UserCommands.CreateCommand c = optionsParser.getUserCommands().createCommand; //QueryResult<User> user = catalogManager.insertUser(new User(c.up.user, c.name, c.email, c.up.password, c.organization, User.Role.USER, "")); QueryResult<User> user = catalogManager.createUser(c.user, c.name, c.email, c.password, c.organization, null, null); System.out.println(createOutput(c.cOpt, user, null)); break; } case "info": { OptionsParser.UserCommands.InfoCommand c = optionsParser.getUserCommands().infoCommand; QueryResult<User> user = catalogManager.getUser( c.up.user != null ? c.up.user : catalogManager.getUserIdBySessionId(sessionId), null, new QueryOptions(c.cOpt.getQueryOptions()), sessionId); System.out.println(createOutput(c.cOpt, user, null)); break; } case "list": { OptionsParser.UserCommands.ListCommand c = optionsParser.getUserCommands().listCommand; String indent = ""; User user = catalogManager.getUser( c.up.user != null ? c.up.user : catalogManager.getUserIdBySessionId(sessionId), null, new QueryOptions("include", Arrays.asList("id", "name", "projects.id", "projects.alias", "projects.name")), sessionId).first(); System.out.println(user.getId() + " - " + user.getName()); indent += "\t"; System.out.println(listProjects(user.getProjects(), c.recursive ? c.level : 1, indent, c.uries, new StringBuilder(), sessionId)); break; } case "login": { OptionsParser.UserCommands.LoginCommand c = optionsParser.getUserCommands().loginCommand; // if (c.up.user == null || c.up.user.isEmpty()) { // throw new CatalogException("Required userId"); // } shellSessionId = sessionId; logoutAtExit = false; if (shellSessionId != null) { shellUserId = c.up.user; } if (sessionFile == null) { sessionFile = new SessionFile(); } sessionFile.setSessionId(sessionId); sessionFile.setUserId(catalogManager.getUserIdBySessionId(sessionId)); saveUserFile(sessionFile); System.out.println(shellSessionId); break; } case "logout": { OptionsParser.UserCommands.LogoutCommand c = optionsParser.getUserCommands().logoutCommand; QueryResult logout; if (c.sessionId == null) { //Logout from interactive shell logout = catalogManager.logout(shellUserId, shellSessionId); shellUserId = null; shellSessionId = null; if (sessionIdFromFile) { sessionFile.setSessionId(null); sessionFile.setUserId(null); saveUserFile(sessionFile); } } else { String userId = catalogManager.getUserIdBySessionId(c.sessionId); logout = catalogManager.logout(userId, c.sessionId); } logoutAtExit = false; System.out.println(logout); break; } default: optionsParser.printUsage(); break; } break; case "projects": switch (optionsParser.getSubCommand()) { case "create": { OptionsParser.ProjectCommands.CreateCommand c = optionsParser.getProjectCommands().createCommand; String user = c.up.user == null || c.up.user.isEmpty() ? catalogManager.getUserIdBySessionId(sessionId) : c.up.user; QueryResult<Project> project = catalogManager.createProject(c.name, c.alias, c.description, c.organization, new QueryOptions(c.cOpt.getQueryOptions()), sessionId); System.out.println(createOutput(c.cOpt, project, null)); break; } case "info": { OptionsParser.ProjectCommands.InfoCommand c = optionsParser.getProjectCommands().infoCommand; long projectId = catalogManager.getProjectId(c.id); QueryResult<Project> project = catalogManager.getProject(projectId, new QueryOptions(c.cOpt.getQueryOptions()), sessionId); System.out.println(createOutput(c.cOpt, project, null)); break; } // case "share": { // OptionsParser.CommandShareResource c = optionsParser.commandShareResource; // // int projectId = catalogManager.getProjectId(c.id); // QueryResult result = catalogManager.shareProject(projectId, new AclEntry(c.user, c.read, c.write, c.execute, c.delete), sessionId); // System.out.println(createOutput(c.cOpt, result, null)); // // break; // } default: optionsParser.printUsage(); break; } break; case "studies": switch (optionsParser.getSubCommand()) { case "create": { OptionsParser.StudyCommands.CreateCommand c = optionsParser.getStudyCommands().createCommand; URI uri = null; if (c.uri != null && !c.uri.isEmpty()) { uri = UriUtils.createUri(c.uri); } Map<File.Bioformat, DataStore> dataStoreMap = parseBioformatDataStoreMap(c); long projectId = catalogManager.getProjectId(c.projectId); ObjectMap attributes = new ObjectMap(); attributes.put(VariantStorageManager.Options.AGGREGATED_TYPE.key(), c.aggregated.toString()); QueryResult<Study> study = catalogManager.createStudy(projectId, c.name, c.alias, c.type, null, c.description, null, null, null, uri, dataStoreMap, null, attributes, new QueryOptions(c.cOpt.getQueryOptions()), sessionId); if (uri != null) { File root = catalogManager.searchFile(study.first().getId(), new Query(FileDBAdaptor.QueryParams.PATH.key(), ""), sessionId).first(); new FileScanner(catalogManager).scan(root, uri, FileScanner.FileScannerPolicy.REPLACE, true, false, sessionId); } System.out.println(createOutput(c.cOpt, study, null)); break; } case "resync": { OptionsParser.StudyCommands.ResyncCommand c = optionsParser.getStudyCommands().resyncCommand; long studyId = catalogManager.getStudyId(c.id); Study study = catalogManager.getStudy(studyId, sessionId).first(); FileScanner fileScanner = new FileScanner(catalogManager); List<File> scan = fileScanner.reSync(study, c.calculateChecksum, sessionId); System.out.println(createOutput(c.cOpt, scan, null)); break; } case "check-files": { OptionsParser.StudyCommands.CheckCommand c = optionsParser.getStudyCommands().checkCommand; long studyId = catalogManager.getStudyId(c.id); Study study = catalogManager.getStudy(studyId, sessionId).first(); FileScanner fileScanner = new FileScanner(catalogManager); List<File> check = fileScanner.checkStudyFiles(study, c.calculateChecksum, sessionId); System.out.println(createOutput(c.cOpt, check, null)); break; } case "info": { OptionsParser.StudyCommands.InfoCommand c = optionsParser.getStudyCommands().infoCommand; long studyId = catalogManager.getStudyId(c.id); QueryResult<Study> study = catalogManager.getStudy(studyId, new QueryOptions(c.cOpt.getQueryOptions()), sessionId); System.out.println(createOutput(c.cOpt, study, null)); break; } case "list": { OptionsParser.StudyCommands.ListCommand c = optionsParser.getStudyCommands().listCommand; long studyId = catalogManager.getStudyId(c.id); List<Study> studies = catalogManager.getStudy(studyId, sessionId).getResult(); String indent = ""; System.out.println(listStudies(studies, c.recursive ? c.level : 1, indent, c.uries, new StringBuilder(), sessionId)); break; } case "status": { OptionsParser.StudyCommands.StatusCommand c = optionsParser.getStudyCommands().statusCommand; long studyId = catalogManager.getStudyId(c.id); Study study = catalogManager.getStudy(studyId, sessionId).first(); FileScanner fileScanner = new FileScanner(catalogManager); /** First, run CheckStudyFiles to find new missing files **/ List<File> checkStudyFiles = fileScanner.checkStudyFiles(study, false, sessionId); List<File> found = checkStudyFiles.stream() .filter(f -> f.getStatus().getName().equals(File.FileStatus.READY)) .collect(Collectors.toList()); int maxFound = found.stream().map(f -> f.getPath().length()).max(Comparator.<Integer>naturalOrder()) .orElse(0); /** Get untracked files **/ // List<URI> untrackedFiles = fileScanner.untrackedFiles(study, sessionId); // // URI studyUri = catalogManager.getStudyUri(studyId); // Map<URI, String> relativeUrisMap = untrackedFiles.stream().collect(Collectors.toMap((k) -> k, (u) -> studyUri.relativize(u).toString())); Map<String, URI> relativeUrisMap = fileScanner.untrackedFiles(study, sessionId); int maxUntracked = relativeUrisMap.keySet().stream().map(String::length) .max(Comparator.<Integer>naturalOrder()).orElse(0); /** Get missing files **/ List<File> missingFiles = catalogManager.getAllFiles(studyId, new Query(FileDBAdaptor.QueryParams.FILE_STATUS.key(), File.FileStatus.MISSING), new QueryOptions(), sessionId).getResult(); int maxMissing = missingFiles.stream().map(f -> f.getPath().length()) .max(Comparator.<Integer>naturalOrder()).orElse(0); /** Print pretty **/ String format = "\t%-" + Math.max(Math.max(maxMissing, maxUntracked), maxFound) + "s -> %s\n"; if (!relativeUrisMap.isEmpty()) { System.out.println("UNTRACKED files"); relativeUrisMap.forEach((s, u) -> System.out.printf(format, s, u)); System.out.println("\n"); } if (!missingFiles.isEmpty()) { System.out.println("MISSING files"); for (File file : missingFiles) { System.out.printf(format, file.getPath(), catalogManager.getFileUri(file)); } System.out.println("\n"); } if (!found.isEmpty()) { System.out.println("FOUND files"); for (File file : found) { System.out.printf(format, file.getPath(), catalogManager.getFileUri(file)); } } break; } case "annotate-variants": { OptionsParser.StudyCommands.AnnotationCommand c = optionsParser .getStudyCommands().annotationCommand; VariantStorage variantStorage = new VariantStorage(catalogManager); long studyId = catalogManager.getStudyId(c.id); long outdirId = catalogManager.getFileId(c.outdir); QueryOptions queryOptions = new QueryOptions(c.cOpt.getQueryOptions()); queryOptions.put(ExecutorManager.EXECUTE, !c.enqueue); queryOptions.add(AnalysisFileIndexer.PARAMETERS, c.dashDashParameters); queryOptions.add(AnalysisFileIndexer.LOG_LEVEL, logLevel); System.out.println(createOutput(c.cOpt, variantStorage.annotateVariants(studyId, outdirId, sessionId, queryOptions), null)); break; } // case "share": { // OptionsParser.CommandShareResource c = optionsParser.commandShareResource; // // int studyId = catalogManager.getStudyId(c.id); // QueryResult result = catalogManager.shareProject(studyId, new AclEntry(c.user, c.read, c.write, c.execute, c.delete), sessionId); // System.out.println(createOutput(c.cOpt, result, null)); // // break; // } default: optionsParser.printUsage(); break; } break; case "files": { switch (optionsParser.getSubCommand()) { case "create": { OptionsParser.FileCommands.CreateCommand c = optionsParser.getFileCommands().createCommand; long studyId = catalogManager.getStudyId(c.studyId); Path inputFile = Paths.get(c.inputFile); URI sourceUri = new URI(null, c.inputFile, null); if (sourceUri.getScheme() == null || sourceUri.getScheme().isEmpty()) { sourceUri = inputFile.toUri(); } if (!catalogManager.getCatalogIOManagerFactory().get(sourceUri).exists(sourceUri)) { throw new IOException("File " + sourceUri + " does not exist"); } QueryResult<File> file = catalogManager.createFile(studyId, c.format, c.bioformat, Paths.get(c.path, inputFile.getFileName().toString()).toString(), c.description, c.parents, -1, sessionId); new CatalogFileUtils(catalogManager).upload(sourceUri, file.first(), null, sessionId, false, false, c.move, c.calculateChecksum); FileMetadataReader.get(catalogManager).setMetadataInformation(file.first(), null, new QueryOptions(c.cOpt.getQueryOptions()), sessionId, false); System.out.println(createOutput(c.cOpt, file, null)); break; } case "create-folder": { OptionsParser.FileCommands.CreateFolderCommand c = optionsParser .getFileCommands().createFolderCommand; long studyId = catalogManager.getStudyId(c.studyId); QueryResult<File> folder = catalogManager.createFolder(studyId, Paths.get(c.path), c.parents, new QueryOptions(c.cOpt.getQueryOptions()), sessionId); System.out.println(createOutput(c.cOpt, folder, null)); break; } case "upload": { OptionsParser.FileCommands.UploadCommand c = optionsParser.getFileCommands().uploadCommand; URI sourceUri = new URI(null, c.inputFile, null); if (sourceUri.getScheme() == null || sourceUri.getScheme().isEmpty()) { sourceUri = Paths.get(c.inputFile).toUri(); } if (!catalogManager.getCatalogIOManagerFactory().get(sourceUri).exists(sourceUri)) { throw new IOException("File " + sourceUri + " does not exist"); } long fileId = catalogManager.getFileId(c.id); QueryResult<File> file = catalogManager.getFile(fileId, new QueryOptions(c.cOpt.getQueryOptions()), sessionId); new CatalogFileUtils(catalogManager).upload(sourceUri, file.first(), null, sessionId, c.replace, c.replace, c.move, c.calculateChecksum); System.out.println(createOutput(c.cOpt, catalogManager.getFile(file.first().getId(), new QueryOptions(c.cOpt.getQueryOptions()), sessionId), null)); break; } case "link": { OptionsParser.FileCommands.LinkCommand c = optionsParser.getFileCommands().linkCommand; Path inputFile = Paths.get(c.inputFile); URI inputUri = UriUtils.createUri(c.inputFile); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(inputUri); if (!ioManager.exists(inputUri)) { throw new FileNotFoundException("File " + inputUri + " not found"); } // long studyId = catalogManager.getStudyId(c.studyId); String path = c.path.isEmpty() ? inputFile.getFileName().toString() : Paths.get(c.path, inputFile.getFileName().toString()).toString(); File file; CatalogFileUtils catalogFileUtils = new CatalogFileUtils(catalogManager); if (ioManager.isDirectory(inputUri)) { ObjectMap params = new ObjectMap("parents", c.parents); file = catalogManager.link(inputUri, c.path, c.studyId, params, sessionId).first(); // file = catalogFileUtils.linkFolder(studyId, path, c.parents, c.description, c.calculateChecksum, inputUri, false, false, sessionId); new FileScanner(catalogManager).scan(file, null, FileScanner.FileScannerPolicy.REPLACE, c.calculateChecksum, false, sessionId); } else { ObjectMap params = new ObjectMap("parents", c.parents); file = catalogManager.link(inputUri, c.path, c.studyId, params, sessionId).first(); // file = catalogManager.createFile(studyId, null, null, // path, c.description, c.parents, -1, sessionId).first(); // file = catalogFileUtils.link(file, c.calculateChecksum, inputUri, false, false, sessionId); file = FileMetadataReader.get(catalogManager).setMetadataInformation(file, null, new QueryOptions(c.cOpt.getQueryOptions()), sessionId, false); } System.out.println(createOutput(c.cOpt, file, null)); break; } case "relink": { OptionsParser.FileCommands.RelinkCommand c = optionsParser.getFileCommands().relinkCommand; Path inputFile = Paths.get(c.inputFile); URI uri = UriUtils.createUri(c.inputFile); if (!inputFile.toFile().exists()) { throw new FileNotFoundException("File " + uri + " not found"); } long fileId = catalogManager.getFileId(c.id, sessionId); File file = catalogManager.getFile(fileId, sessionId).first(); new CatalogFileUtils(catalogManager).link(file, c.calculateChecksum, uri, false, true, sessionId); file = catalogManager.getFile(file.getId(), new QueryOptions(c.cOpt.getQueryOptions()), sessionId) .first(); file = FileMetadataReader.get(catalogManager).setMetadataInformation(file, null, new QueryOptions(c.cOpt.getQueryOptions()), sessionId, false); System.out.println(createOutput(c.cOpt, file, null)); break; } case "refresh": { OptionsParser.FileCommands.RefreshCommand c = optionsParser.getFileCommands().refreshCommand; long fileId = catalogManager.getFileId(c.id); File file = catalogManager.getFile(fileId, sessionId).first(); List<File> files; QueryOptions queryOptions = new QueryOptions(c.cOpt.getQueryOptions()); CatalogFileUtils catalogFileUtils = new CatalogFileUtils(catalogManager); FileMetadataReader fileMetadataReader = FileMetadataReader.get(catalogManager); if (file.getType() == File.Type.FILE) { File file1 = catalogFileUtils.checkFile(file, false, sessionId); file1 = fileMetadataReader.setMetadataInformation(file1, null, queryOptions, sessionId, false); if (file == file1) { //If the file is the same, it was not modified. Only return modified files. files = Collections.emptyList(); } else { files = Collections.singletonList(file); } } else { List<File> result = catalogManager.getAllFilesInFolder(file.getId(), null, sessionId) .getResult(); files = new ArrayList<>(result.size()); for (File f : result) { File file1 = fileMetadataReader.setMetadataInformation(f, null, queryOptions, sessionId, false); if (f != file1) { //Add only modified files. files.add(file1); } } } System.out.println(createOutput(c.cOpt, files, null)); break; } case "info": { OptionsParser.FileCommands.InfoCommand c = optionsParser.getFileCommands().infoCommand; long fileId = catalogManager.getFileId(c.id); QueryResult<File> file = catalogManager.getFile(fileId, new QueryOptions(c.cOpt.getQueryOptions()), sessionId); System.out.println(createOutput(optionsParser.getCommonOptions(), file, null)); break; } case "search": { OptionsParser.FileCommands.SearchCommand c = optionsParser.getFileCommands().searchCommand; long studyId = catalogManager.getStudyId(c.studyId); Query query = new Query(); if (c.name != null) query.put(FileDBAdaptor.QueryParams.NAME.key(), "~" + c.name); if (c.directory != null) query.put(FileDBAdaptor.QueryParams.DIRECTORY.key(), c.directory); if (c.bioformats != null) query.put(FileDBAdaptor.QueryParams.BIOFORMAT.key(), c.bioformats); if (c.types != null) query.put(FileDBAdaptor.QueryParams.TYPE.key(), c.types); if (c.status != null) query.put(FileDBAdaptor.QueryParams.STATUS_NAME.key(), c.status); QueryResult<File> fileQueryResult = catalogManager.searchFile(studyId, query, new QueryOptions(c.cOpt.getQueryOptions()), sessionId); System.out.println(createOutput(optionsParser.getCommonOptions(), fileQueryResult, null)); break; } case "list": { OptionsParser.FileCommands.ListCommand c = optionsParser.getFileCommands().listCommand; long fileId = catalogManager.getFileId(c.id); List<File> result = catalogManager.getFile(fileId, sessionId).getResult(); long studyId = catalogManager.getStudyIdByFileId(fileId); System.out.println(listFiles(result, studyId, c.recursive ? c.level : 1, "", c.uries, new StringBuilder(), sessionId)); break; } case "index": { OptionsParser.FileCommands.IndexCommand c = optionsParser.getFileCommands().indexCommand; AnalysisFileIndexer analysisFileIndexer = new AnalysisFileIndexer(catalogManager); long fileId = catalogManager.getFileId(c.id); long outdirId = catalogManager.getFileId(c.outdir); if (outdirId < 0) { outdirId = catalogManager.getFileParent(fileId, null, sessionId).first().getId(); } String sid = sessionId; QueryOptions queryOptions = new QueryOptions(c.cOpt.getQueryOptions()); if (c.enqueue) { queryOptions.put(ExecutorManager.EXECUTE, false); if (c.up.sessionId == null || c.up.sessionId.isEmpty()) { sid = login(c.up); } } else { queryOptions.add(ExecutorManager.EXECUTE, true); } queryOptions.put(AnalysisFileIndexer.TRANSFORM, c.transform); queryOptions.put(AnalysisFileIndexer.LOAD, c.load); queryOptions.add(AnalysisFileIndexer.PARAMETERS, c.dashDashParameters); queryOptions.add(AnalysisFileIndexer.LOG_LEVEL, logLevel); queryOptions.add(VariantStorageManager.Options.CALCULATE_STATS.key(), c.calculateStats); queryOptions.add(VariantStorageManager.Options.ANNOTATE.key(), c.annotate); logger.debug("logLevel: {}", logLevel); QueryResult<Job> queryResult = analysisFileIndexer.index(fileId, outdirId, sid, queryOptions); System.out.println(createOutput(c.cOpt, queryResult, null)); break; } default: optionsParser.printUsage(); break; } break; } case "samples": { switch (optionsParser.getSubCommand()) { case "info": { OptionsParser.SampleCommands.InfoCommand c = optionsParser.sampleCommands.infoCommand; QueryResult<Sample> sampleQueryResult = catalogManager.getSample(c.id, new QueryOptions(c.cOpt.getQueryOptions()), sessionId); System.out.println(createOutput(c.cOpt, sampleQueryResult, null)); break; } case "search": { OptionsParser.SampleCommands.SearchCommand c = optionsParser.sampleCommands.searchCommand; long studyId = catalogManager.getStudyId(c.studyId); QueryOptions queryOptions = new QueryOptions(c.cOpt.getQueryOptions()); Query query = new Query(); if (c.sampleIds != null && !c.sampleIds.isEmpty()) { query.append(SampleDBAdaptor.QueryParams.ID.key(), c.sampleIds); } if (c.sampleNames != null && !c.sampleNames.isEmpty()) { query.append(SampleDBAdaptor.QueryParams.NAME.key(), c.sampleNames); } if (c.annotation != null && !c.annotation.isEmpty()) { for (String s : c.annotation) { String[] strings = org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptorUtils .splitOperator(s); query.append(SampleDBAdaptor.QueryParams.ANNOTATION.key() + "." + strings[0], strings[1] + strings[2]); } } if (c.variableSetId != null && !c.variableSetId.isEmpty()) { query.append(SampleDBAdaptor.QueryParams.VARIABLE_SET_ID.key(), c.variableSetId); } QueryResult<Sample> sampleQueryResult = catalogManager.getAllSamples(studyId, query, queryOptions, sessionId); System.out.println(createOutput(c.cOpt, sampleQueryResult, null)); break; } case "load": { OptionsParser.SampleCommands.LoadCommand c = optionsParser.sampleCommands.loadCommand; CatalogSampleAnnotationsLoader catalogSampleAnnotationsLoader = new CatalogSampleAnnotationsLoader( catalogManager); long fileId = catalogManager.getFileId(c.pedigreeFileId); File pedigreeFile = catalogManager.getFile(fileId, sessionId).first(); QueryResult<Sample> sampleQueryResult = catalogSampleAnnotationsLoader.loadSampleAnnotations( pedigreeFile, c.variableSetId == 0 ? null : c.variableSetId, sessionId); System.out.println(createOutput(c.cOpt, sampleQueryResult, null)); break; } case "delete": { OptionsParser.SampleCommands.DeleteCommand c = optionsParser.sampleCommands.deleteCommand; QueryResult<Sample> sampleQueryResult = catalogManager.deleteSample(c.id, new QueryOptions(c.cOpt.getQueryOptions()), sessionId); System.out.println(createOutput(c.cOpt, sampleQueryResult, null)); break; } default: { optionsParser.printUsage(); break; } } break; } case "cohorts": { switch (optionsParser.getSubCommand()) { case OptionsParser.CohortCommands.InfoCommand.COMMAND_NAME: { OptionsParser.CohortCommands.InfoCommand c = optionsParser.cohortCommands.infoCommand; QueryResult<Cohort> cohortQueryResult = catalogManager.getCohort(c.id, new QueryOptions(c.cOpt.getQueryOptions()), sessionId); System.out.println(createOutput(c.cOpt, cohortQueryResult, null)); break; } case OptionsParser.CohortCommands.SamplesCommand.COMMAND_NAME: { OptionsParser.CohortCommands.SamplesCommand c = optionsParser.cohortCommands.samplesCommand; Cohort cohort = catalogManager.getCohort(c.id, null, sessionId).first(); QueryOptions queryOptions = new QueryOptions(c.cOpt.getQueryOptions()); Query query = new Query(SampleDBAdaptor.QueryParams.ID.key(), cohort.getSamples()); QueryResult<Sample> sampleQueryResult = catalogManager.getAllSamples( catalogManager.getStudyIdByCohortId(cohort.getId()), query, queryOptions, sessionId); OptionsParser.CommonOptions cOpt = c.cOpt; StringBuilder sb = createOutput(cOpt, sampleQueryResult, null); System.out.println(sb.toString()); break; } case OptionsParser.CohortCommands.CreateCommand.COMMAND_NAME: { OptionsParser.CohortCommands.CreateCommand c = optionsParser.cohortCommands.createCommand; Map<String, List<Sample>> cohorts = new HashMap<>(); long studyId = catalogManager.getStudyId(c.studyId); if (c.sampleIds != null && !c.sampleIds.isEmpty()) { QueryOptions queryOptions = new QueryOptions("include", "projects.studies.samples.id"); Query query = new Query(SampleDBAdaptor.QueryParams.ID.key(), c.sampleIds); // queryOptions.put("variableSetId", c.variableSetId); QueryResult<Sample> sampleQueryResult = catalogManager.getAllSamples(studyId, query, queryOptions, sessionId); cohorts.put(c.name, sampleQueryResult.getResult()); } else if (StringUtils.isNotEmpty(c.tagmap)) { List<QueryResult<Cohort>> queryResults = createCohorts(sessionId, studyId, c.tagmap, catalogManager, logger); System.out.println(createOutput(c.cOpt, queryResults, null)); } else { // QueryOptions queryOptions = c.cOpt.getQueryOptions(); // queryOptions.put("annotation", c.annotation); final long variableSetId; final VariableSet variableSet; if (StringUtils.isNumeric(c.variableSet)) { variableSetId = Long.parseLong(c.variableSet); variableSet = catalogManager.getVariableSet(variableSetId, null, sessionId).first(); } else if (StringUtils.isEmpty(c.variableSet)) { List<VariableSet> variableSets = catalogManager.getStudy(studyId, new QueryOptions("include", "projects.studies.variableSets"), sessionId).first() .getVariableSets(); if (!variableSets.isEmpty()) { variableSet = variableSets.get(0); //Get the first VariableSetId variableSetId = variableSet.getId(); } else { throw new CatalogException("Expected variableSetId"); } } else { QueryOptions query = new QueryOptions(StudyDBAdaptor.VariableSetParams.NAME.key(), c.variableSet); variableSet = catalogManager.getAllVariableSet(studyId, query, sessionId).first(); if (variableSet == null) { throw new CatalogException("Variable set \"" + c.variableSet + "\" not found"); } variableSetId = variableSet.getId(); } c.name = ((c.name == null) || c.name.isEmpty()) ? "" : (c.name + "."); for (Variable variable : variableSet.getVariables()) { if (variable.getName().equals(c.variable)) { for (String value : variable.getAllowedValues()) { QueryOptions queryOptions = new QueryOptions(c.cOpt.getQueryOptions()); Query query = new Query( SampleDBAdaptor.QueryParams.ANNOTATION.key() + "." + c.variable, value) .append(SampleDBAdaptor.QueryParams.VARIABLE_SET_ID.key(), variableSetId); QueryResult<Sample> sampleQueryResult = catalogManager.getAllSamples(studyId, query, queryOptions, sessionId); cohorts.put(c.name + value, sampleQueryResult.getResult()); } } } if (cohorts.isEmpty()) { logger.error("VariableSetId {} does not contain any variable with id = {}.", variableSetId, c.variable); returnValue = 2; } } if (!cohorts.isEmpty()) { List<QueryResult<Cohort>> queryResults = new ArrayList<>(cohorts.size()); for (Map.Entry<String, List<Sample>> entry : cohorts.entrySet()) { List<Long> sampleIds = new LinkedList<>(); for (Sample sample : entry.getValue()) { sampleIds.add(sample.getId()); } QueryResult<Cohort> cohort = catalogManager.createCohort(studyId, entry.getKey(), c.type, c.description, sampleIds, c.cOpt.getQueryOptions(), sessionId); queryResults.add(cohort); } System.out.println(createOutput(c.cOpt, queryResults, null)); } // System.out.println(createSamplesOutput(c.cOpt, sampleQueryResult)); break; } case OptionsParser.CohortCommands.StatsCommand.COMMAND_NAME: { OptionsParser.CohortCommands.StatsCommand c = optionsParser.cohortCommands.statsCommand; VariantStorage variantStorage = new VariantStorage(catalogManager); long outdirId = catalogManager.getFileId(c.outdir); QueryOptions queryOptions = new QueryOptions(c.cOpt.getQueryOptions()); if (c.enqueue) { queryOptions.put(ExecutorManager.EXECUTE, false); } else { queryOptions.add(ExecutorManager.EXECUTE, true); } queryOptions.add(AnalysisFileIndexer.PARAMETERS, c.dashDashParameters); queryOptions.add(AnalysisFileIndexer.LOG_LEVEL, logLevel); if (c.tagmap != null) { queryOptions.put(VariantStorageManager.Options.AGGREGATION_MAPPING_PROPERTIES.key(), c.tagmap); } else if (c.cohortIds == null) { logger.error("--cohort-id nor --aggregation-mapping-file provided"); throw new IllegalArgumentException( "--cohort-id or --aggregation-mapping-file is required to specify cohorts"); } System.out.println(createOutput(c.cOpt, variantStorage.calculateStats(outdirId, c.cohortIds, sessionId, queryOptions), null)); break; } default: { optionsParser.printUsage(); break; } } break; } case "jobs": { switch (optionsParser.getSubCommand()) { case "info": { OptionsParser.JobsCommands.InfoCommand c = optionsParser.getJobsCommands().infoCommand; QueryResult<Job> jobQueryResult = catalogManager.getJob(c.id, new QueryOptions(c.cOpt.getQueryOptions()), sessionId); System.out.println(createOutput(c.cOpt, jobQueryResult, null)); break; } case "finished": { OptionsParser.JobsCommands.DoneJobCommand c = optionsParser.getJobsCommands().doneJobCommand; QueryResult<Job> jobQueryResult = catalogManager.getJob(c.id, new QueryOptions(c.cOpt.getQueryOptions()), sessionId); Job job = jobQueryResult.first(); if (c.force) { if (job.getStatus().getName().equals(Job.JobStatus.ERROR) || job.getStatus().getName().equals(Job.JobStatus.READY)) { logger.info("Job status is '{}' . Nothing to do.", job.getStatus().getName()); System.out.println(createOutput(c.cOpt, jobQueryResult, null)); } } else if (!job.getStatus().getName().equals(Job.JobStatus.DONE)) { throw new Exception("Job status != DONE. Need --force to continue"); } /** Record output **/ ExecutionOutputRecorder outputRecorder = new ExecutionOutputRecorder(catalogManager, sessionId); if (c.discardOutput) { String tempJobsDir = catalogManager.getCatalogConfiguration().getTempJobsDir(); URI tmpOutDirUri = IndexDaemon.getJobTemporaryFolder(job.getId(), tempJobsDir).toUri(); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(tmpOutDirUri); if (ioManager.exists(tmpOutDirUri)) { logger.info("Deleting temporal job output folder: {}", tmpOutDirUri); ioManager.deleteDirectory(tmpOutDirUri); } else { logger.info("Temporal job output folder already removed: {}", tmpOutDirUri); } } else { outputRecorder.recordJobOutput(job); } outputRecorder.postProcessJob(job, c.error); /** Change status to ERROR or READY **/ ObjectMap parameters = new ObjectMap(); if (c.error) { parameters.put("status.name", Job.JobStatus.ERROR); parameters.put("error", Job.ERRNO_ABORTED); parameters.put("errorDescription", Job.ERROR_DESCRIPTIONS.get(Job.ERRNO_ABORTED)); } else { parameters.put("status.name", Job.JobStatus.READY); } catalogManager.modifyJob(job.getId(), parameters, sessionId); jobQueryResult = catalogManager.getJob(c.id, new QueryOptions(c.cOpt.getQueryOptions()), sessionId); System.out.println(createOutput(c.cOpt, jobQueryResult, null)); break; } case "status": { OptionsParser.JobsCommands.StatusCommand c = optionsParser.getJobsCommands().statusCommand; final List<Long> studyIds; if (c.studyId == null || c.studyId.isEmpty()) { studyIds = catalogManager .getAllStudies(new Query(), new QueryOptions("include", "id"), sessionId).getResult() .stream().map(Study::getId).collect(Collectors.toList()); } else { studyIds = new LinkedList<>(); for (String s : c.studyId.split(",")) { studyIds.add(catalogManager.getStudyId(s)); } } for (Long studyId : studyIds) { QueryResult<Job> allJobs = catalogManager.getAllJobs(studyId, new Query(JobDBAdaptor.QueryParams.STATUS_NAME.key(), Collections.singletonList(Job.JobStatus.RUNNING.toString())), new QueryOptions(), sessionId); for (Iterator<Job> iterator = allJobs.getResult().iterator(); iterator.hasNext();) { Job job = iterator.next(); System.out.format("Job - %s [%d] - %s\n", job.getName(), job.getId(), job.getDescription()); // URI tmpOutDirUri = job.getTmpOutDirUri(); String tempJobsDir = catalogManager.getCatalogConfiguration().getTempJobsDir(); URI tmpOutDirUri = IndexDaemon.getJobTemporaryFolder(job.getId(), tempJobsDir).toUri(); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(tmpOutDirUri); try { ioManager.listFilesStream(tmpOutDirUri).sorted().forEach(uri -> { String count; try { long fileSize = ioManager.getFileSize(uri); count = humanReadableByteCount(fileSize, false); } catch (CatalogIOException e) { count = "ERROR"; } System.out.format("\t%s [%s]\n", tmpOutDirUri.relativize(uri), count); }); } catch (CatalogIOException e) { System.out.println("Unable to read files from " + tmpOutDirUri + " - " + e.getCause().getMessage()); } if (iterator.hasNext()) { System.out.println("-----------------------------------------"); } } } break; } case "run": { OptionsParser.JobsCommands.RunJobCommand c = optionsParser.getJobsCommands().runJobCommand; long studyId = catalogManager.getStudyId(c.studyId); long outdirId = catalogManager.getFileId(c.outdir); long toolId = catalogManager.getToolId(c.toolId); String toolName; ToolManager toolManager; if (toolId < 0) { toolManager = new ToolManager(c.toolId, c.execution); //LEGACY MODE, AVOID USING toolName = c.toolId; } else { Tool tool = catalogManager.getTool(toolId, sessionId).getResult().get(0); toolManager = new ToolManager(Paths.get(tool.getPath()).getParent(), tool.getName(), c.execution); toolName = tool.getName(); } List<Long> inputFiles = new LinkedList<>(); Map<String, List<String>> localParams = new HashMap<>(); for (String key : c.params.keySet()) { localParams.put(key, c.params.getAsStringList(key)); } Execution ex = toolManager.getExecution(); // Set input param for (InputParam inputParam : ex.getInputParams()) { if (c.params.containsKey(inputParam.getName())) { List<String> filePaths = new LinkedList<>(); for (String fileId : c.params.getAsStringList(inputParam.getName())) { File file = catalogManager.getFile(catalogManager.getFileId(fileId), sessionId) .getResult().get(0); filePaths.add(catalogManager.getFileUri(file).getPath()); inputFiles.add(file.getId()); } localParams.put(inputParam.getName(), filePaths); } } // Set outdir String outputParam = toolManager.getExecution().getOutputParam(); File outdir = catalogManager.getFile(outdirId, sessionId).first(); localParams.put(outputParam, Collections.singletonList(catalogManager.getFileUri(outdir).getPath())); QueryResult<Job> jobQueryResult = new JobFactory(catalogManager).createJob(toolManager, localParams, studyId, c.name, c.description, outdir, inputFiles, sessionId, true); System.out.println(createOutput(c.cOpt, jobQueryResult, null)); break; } default: { optionsParser.printUsage(); break; } } break; } case "tools": { switch (optionsParser.getSubCommand()) { case "create": { OptionsParser.ToolCommands.CreateCommand c = optionsParser.getToolCommands().createCommand; Path path = Paths.get(c.path); FileUtils.checkDirectory(path); QueryResult<Tool> tool = catalogManager.createTool(c.alias, c.description, null, null, path.toAbsolutePath().toString(), c.openTool, sessionId); System.out.println(createOutput(c.cOpt, tool, null)); break; } case "info": { OptionsParser.ToolCommands.InfoCommand c = optionsParser.getToolCommands().infoCommand; long toolId = catalogManager.getToolId(c.id); ToolManager toolManager; String toolName; if (toolId < 0) { toolManager = new ToolManager(c.id, null); //LEGACY MODE, AVOID USING toolName = c.id; System.out.println(createOutput(c.cOpt, toolManager.getManifest(), null)); } else { Tool tool = catalogManager.getTool(toolId, sessionId).getResult().get(0); toolManager = new ToolManager(Paths.get(tool.getPath()).getParent(), tool.getName(), null); toolName = tool.getName(); System.out.println(createOutput(c.cOpt, tool, null)); } break; } default: { optionsParser.printUsage(); break; } } break; } case "exit": { } break; case "help": default: optionsParser.printUsage(); // logger.info("Unknown command"); break; } logout(sessionId); return returnValue; }
From source file:org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageEngine.java
@Override public void dropFile(String study, int fileId) throws StorageEngineException { ObjectMap options = configuration.getStorageEngine(STORAGE_ENGINE_ID).getVariant().getOptions(); // Use ETL as helper class AbstractHadoopVariantStoragePipeline etl = newStoragePipeline(true); VariantDBAdaptor dbAdaptor = etl.getDBAdaptor(); StudyConfiguration studyConfiguration; StudyConfigurationManager scm = dbAdaptor.getStudyConfigurationManager(); List<Integer> fileList = Collections.singletonList(fileId); final int studyId; if (StringUtils.isNumeric(study)) { studyId = Integer.parseInt(study); } else {/*www . j a va 2s . c o m*/ studyConfiguration = scm.getStudyConfiguration(study, null).first(); studyId = studyConfiguration.getStudyId(); } // Pre delete long lock = scm.lockStudy(studyId); try { studyConfiguration = scm.getStudyConfiguration(studyId, null).first(); if (!studyConfiguration.getIndexedFiles().contains(fileId)) { throw StorageEngineException.unableToExecute("File not indexed.", fileId, studyConfiguration); } boolean resume = options.getBoolean(Options.RESUME.key(), Options.RESUME.defaultValue()) || options.getBoolean(HadoopVariantStorageEngine.HADOOP_LOAD_VARIANT_RESUME, false); BatchFileOperation operation = etl.addBatchOperation(studyConfiguration, VariantTableDeletionDriver.JOB_OPERATION_NAME, fileList, resume, BatchFileOperation.Type.REMOVE); options.put(AbstractVariantTableDriver.TIMESTAMP, operation.getTimestamp()); scm.updateStudyConfiguration(studyConfiguration, null); } finally { scm.unLockStudy(studyId, lock); } // Delete Thread hook = etl.newShutdownHook(VariantTableDeletionDriver.JOB_OPERATION_NAME, fileList); try { Runtime.getRuntime().addShutdownHook(hook); String archiveTable = getArchiveTableName(studyId, options); HBaseCredentials variantsTable = getDbCredentials(); String hadoopRoute = options.getString(HADOOP_BIN, "hadoop"); String jar = AbstractHadoopVariantStoragePipeline.getJarWithDependencies(options); Class execClass = VariantTableDeletionDriver.class; String args = VariantTableDeletionDriver.buildCommandLineArgs(variantsTable.toString(), archiveTable, variantsTable.getTable(), studyId, fileList, options); String executable = hadoopRoute + " jar " + jar + ' ' + execClass.getName(); long startTime = System.currentTimeMillis(); logger.info("------------------------------------------------------"); logger.info("Remove file ID {} in archive '{}' and analysis table '{}'", fileId, archiveTable, variantsTable.getTable()); logger.debug(executable + " " + args); logger.info("------------------------------------------------------"); int exitValue = getMRExecutor(options).run(executable, args); logger.info("------------------------------------------------------"); logger.info("Exit value: {}", exitValue); logger.info("Total time: {}s", (System.currentTimeMillis() - startTime) / 1000.0); if (exitValue != 0) { throw new StorageEngineException("Error removing fileId " + fileId + " from tables "); } // Post Delete // If everything went fine, remove file column from Archive table and from studyconfig lock = scm.lockStudy(studyId); try { studyConfiguration = scm.getStudyConfiguration(studyId, null).first(); etl.secureSetStatus(studyConfiguration, BatchFileOperation.Status.READY, VariantTableDeletionDriver.JOB_OPERATION_NAME, fileList); studyConfiguration.getIndexedFiles().remove(fileId); scm.updateStudyConfiguration(studyConfiguration, null); } finally { scm.unLockStudy(studyId, lock); } } catch (Exception e) { etl.setStatus(BatchFileOperation.Status.ERROR, VariantTableDeletionDriver.JOB_OPERATION_NAME, fileList); throw e; } finally { Runtime.getRuntime().removeShutdownHook(hook); } }
From source file:org.opendatakit.common.android.database.DataModelDatabaseHelper.java
/** * Accessor to retrieve the database tableId given a formId * * @param db/*from ww w . j ava 2 s . c o m*/ * @param formId * -- either the integer _ID or the textual form_id * @return */ public static IdInstanceNameStruct getIds(SQLiteDatabase db, String formId) { boolean isNumericId = StringUtils.isNumeric(formId); Cursor c = null; try { c = db.query(FORMS_TABLE_NAME, new String[] { FormsColumns._ID, FormsColumns.FORM_ID, FormsColumns.TABLE_ID, FormsColumns.INSTANCE_NAME }, (isNumericId ? FormsColumns._ID : FormsColumns.FORM_ID) + "=?", new String[] { formId }, null, null, null); if (c.moveToFirst()) { int idxId = c.getColumnIndex(FormsColumns._ID); int idxFormId = c.getColumnIndex(FormsColumns.FORM_ID); int idxTableId = c.getColumnIndex(FormsColumns.TABLE_ID); int idxInstanceName = c.getColumnIndex(FormsColumns.INSTANCE_NAME); return new IdInstanceNameStruct(c.getInt(idxId), c.getString(idxFormId), c.getString(idxTableId), c.isNull(idxInstanceName) ? null : c.getString(idxInstanceName)); } } finally { if (c != null && !c.isClosed()) { c.close(); } } return null; }
From source file:org.opendatakit.common.android.database.IdInstanceNameStruct.java
/** * Accessor to retrieve the database tableId given a formId * * @param db/*from w w w.jav a 2 s . c o m*/ * @param formId * -- either the integer _ID or the textual form_id * @return */ public static IdInstanceNameStruct getIds(SQLiteDatabase db, String formId) { boolean isNumericId = StringUtils.isNumeric(formId); Cursor c = null; try { c = db.query(DatabaseConstants.FORMS_TABLE_NAME, new String[] { FormsColumns._ID, FormsColumns.FORM_ID, FormsColumns.TABLE_ID, FormsColumns.INSTANCE_NAME }, (isNumericId ? FormsColumns._ID : FormsColumns.FORM_ID) + "=?", new String[] { formId }, null, null, null); if (c.moveToFirst()) { int idxId = c.getColumnIndex(FormsColumns._ID); int idxFormId = c.getColumnIndex(FormsColumns.FORM_ID); int idxTableId = c.getColumnIndex(FormsColumns.TABLE_ID); int idxInstanceName = c.getColumnIndex(FormsColumns.INSTANCE_NAME); return new IdInstanceNameStruct(c.getInt(idxId), ODKDatabaseUtils.get().getIndexAsString(c, idxFormId), ODKDatabaseUtils.get().getIndexAsString(c, idxTableId), ODKDatabaseUtils.get().getIndexAsString(c, idxInstanceName)); } } finally { if (c != null && !c.isClosed()) { c.close(); } } return null; }
From source file:org.opendatakit.common.android.provider.impl.FormsProviderImpl.java
@Override public Cursor query(Uri uri, String[] projection, String where, String[] whereArgs, String sortOrder) { List<String> segments = uri.getPathSegments(); if (segments.size() < 1 || segments.size() > 2) { throw new IllegalArgumentException("Unknown URI (incorrect number of segments!) " + uri); }//from www. j a va 2 s .c om String appName = segments.get(0); ODKFileUtils.verifyExternalStorageAvailability(); ODKFileUtils.assertDirectoryStructure(appName); WebLogger log = WebLogger.getLogger(appName); String uriFormId = ((segments.size() == 2) ? segments.get(1) : null); boolean isNumericId = StringUtils.isNumeric(uriFormId); // Modify the where clause to account for the presence of // a form id. Accept either: // (1) numeric _ID value // (2) string FORM_ID value. String whereId; String[] whereIdArgs; if (uriFormId == null) { whereId = where; whereIdArgs = whereArgs; } else { if (TextUtils.isEmpty(where)) { whereId = (isNumericId ? FormsColumns._ID : FormsColumns.FORM_ID) + "=?"; whereIdArgs = new String[1]; whereIdArgs[0] = uriFormId; } else { whereId = (isNumericId ? FormsColumns._ID : FormsColumns.FORM_ID) + "=? AND (" + where + ")"; whereIdArgs = new String[whereArgs.length + 1]; whereIdArgs[0] = uriFormId; for (int i = 0; i < whereArgs.length; ++i) { whereIdArgs[i + 1] = whereArgs[i]; } } } // Get the database and run the query SQLiteDatabase db = null; boolean success = false; Cursor c = null; try { db = DatabaseFactory.get().getDatabase(getContext(), appName); c = db.query(DatabaseConstants.FORMS_TABLE_NAME, projection, whereId, whereIdArgs, null, null, sortOrder); success = true; } catch (Exception e) { log.w(t, "Unable to query database for appName: " + appName); return null; } finally { if (!success && db != null) { db.close(); } } if (c == null) { log.w(t, "Unable to query database for appName: " + appName); return null; } // Tell the cursor what uri to watch, so it knows when its source data // changes c.setNotificationUri(getContext().getContentResolver(), uri); return c; }