List of usage examples for java.util TreeMap get
public V get(Object key)
From source file:edu.mit.viral.shen.DroidFish.java
private final Dialog moveListMenuDialog() { final int EDIT_HEADERS = 0; final int EDIT_COMMENTS = 1; final int REMOVE_SUBTREE = 2; final int MOVE_VAR_UP = 3; final int MOVE_VAR_DOWN = 4; final int ADD_NULL_MOVE = 5; List<CharSequence> lst = new ArrayList<CharSequence>(); List<Integer> actions = new ArrayList<Integer>(); // lst.add(getString(R.string.edit_headers)); actions.add(EDIT_HEADERS); // if (ctrl.humansTurn()) { lst.add(getString(R.string.edit_comments)); actions.add(EDIT_COMMENTS);/*from www. j a v a 2 s . c om*/ // } // lst.add(getString(R.string.truncate_gametree)); actions.add(REMOVE_SUBTREE); // if (ctrl.numVariations() > 1) { // lst.add(getString(R.string.move_var_up)); actions.add(MOVE_VAR_UP); // lst.add(getString(R.string.move_var_down)); actions.add(MOVE_VAR_DOWN); // } boolean allowNullMove = gameMode.analysisMode() || (gameMode.playerWhite() && gameMode.playerBlack() && !gameMode.clocksActive()); if (allowNullMove) { lst.add(getString(R.string.add_null_move)); actions.add(ADD_NULL_MOVE); } final List<Integer> finalActions = actions; AlertDialog.Builder builder = new AlertDialog.Builder(this); builder.setTitle(R.string.edit_game); builder.setItems(lst.toArray(new CharSequence[lst.size()]), new DialogInterface.OnClickListener() { public void onClick(DialogInterface dialog, int item) { switch (finalActions.get(item)) { case EDIT_HEADERS: { final TreeMap<String, String> headers = new TreeMap<String, String>(); ctrl.getHeaders(headers); AlertDialog.Builder builder = new AlertDialog.Builder(DroidFish.this); builder.setTitle(R.string.edit_headers); View content = View.inflate(DroidFish.this, R.layout.edit_headers, null); builder.setView(content); final TextView event, site, date, round, white, black; event = (TextView) content.findViewById(R.id.ed_header_event); site = (TextView) content.findViewById(R.id.ed_header_site); date = (TextView) content.findViewById(R.id.ed_header_date); round = (TextView) content.findViewById(R.id.ed_header_round); white = (TextView) content.findViewById(R.id.ed_header_white); black = (TextView) content.findViewById(R.id.ed_header_black); event.setText(headers.get("Event")); site.setText(headers.get("Site")); date.setText(headers.get("Date")); round.setText(headers.get("Round")); white.setText(headers.get("White")); black.setText(headers.get("Black")); builder.setNegativeButton(R.string.cancel, null); builder.setPositiveButton(android.R.string.ok, new Dialog.OnClickListener() { public void onClick(DialogInterface dialog, int which) { headers.put("Event", event.getText().toString().trim()); headers.put("Site", site.getText().toString().trim()); headers.put("Date", date.getText().toString().trim()); headers.put("Round", round.getText().toString().trim()); headers.put("White", white.getText().toString().trim()); headers.put("Black", black.getText().toString().trim()); ctrl.setHeaders(headers); setBoardFlip(true); } }); builder.show(); break; } case EDIT_COMMENTS: { AlertDialog.Builder builder = new AlertDialog.Builder(DroidFish.this); builder.setTitle(R.string.edit_comments); View content = View.inflate(DroidFish.this, R.layout.edit_comments, null); builder.setView(content); DroidChessController.CommentInfo commInfo = ctrl.getComments(); final TextView preComment, moveView, nag, postComment; preComment = (TextView) content.findViewById(R.id.ed_comments_pre); moveView = (TextView) content.findViewById(R.id.ed_comments_move); nag = (TextView) content.findViewById(R.id.ed_comments_nag); postComment = (TextView) content.findViewById(R.id.ed_comments_post); preComment.setText(commInfo.preComment); postComment.setText(commInfo.postComment); moveView.setText(commInfo.move); String nagStr = Node.nagStr(commInfo.nag).trim(); if ((nagStr.length() == 0) && (commInfo.nag > 0)) nagStr = String.format(Locale.US, "%d", commInfo.nag); nag.setText(nagStr); builder.setNegativeButton(R.string.cancel, null); builder.setPositiveButton(android.R.string.ok, new Dialog.OnClickListener() { public void onClick(DialogInterface dialog, int which) { String pre = preComment.getText().toString().trim(); String post = postComment.getText().toString().trim(); int nagVal = Node.strToNag(nag.getText().toString()); DroidChessController.CommentInfo commInfo = new DroidChessController.CommentInfo(); commInfo.preComment = pre; commInfo.postComment = post; commInfo.nag = nagVal; ctrl.setComments(commInfo); } }); builder.show(); break; } case REMOVE_SUBTREE: ctrl.removeSubTree(); break; case MOVE_VAR_UP: ctrl.moveVariation(-1); break; case MOVE_VAR_DOWN: ctrl.moveVariation(1); break; case ADD_NULL_MOVE: ctrl.makeHumanNullMove(); break; } moveListMenuDlg = null; } }); AlertDialog alert = builder.create(); moveListMenuDlg = alert; return alert; }
From source file:net.spfbl.spf.SPF.java
public static TreeSet<String> clear(String token) { TreeSet<String> clearSet = new TreeSet<String>(); TreeMap<String, Distribution> distribuitonMap = CacheDistribution.getAll(token); for (String key : distribuitonMap.keySet()) { Distribution distribution = distribuitonMap.get(key); if (distribution != null) { if (distribution.clear()) { clearSet.add(key);//from w w w .java 2s .c o m distribution.getStatus(token); Peer.sendToAll(key, distribution); } } if (Block.dropExact(key)) { clearSet.add(key); } } for (String key : Block.getAllTokens(token)) { if (Block.dropExact(key)) { clearSet.add(key); } } // for (String key : Peer.clearAllReputation(token)) { // clearSet.add(key); // } return clearSet; }
From source file:org.apache.catalina.core.StandardContext.java
/** * Load and initialize all servlets marked "load on startup" in the * web application deployment descriptor. * * @param children Array of wrappers for all currently defined * servlets (including those not declared load on startup) */// ww w . j a v a 2s. com public void loadOnStartup(Container children[]) { // Collect "load on startup" servlets that need to be initialized TreeMap map = new TreeMap(); for (int i = 0; i < children.length; i++) { Wrapper wrapper = (Wrapper) children[i]; int loadOnStartup = wrapper.getLoadOnStartup(); if (loadOnStartup < 0) continue; if (loadOnStartup == 0) // Arbitrarily put them last loadOnStartup = Integer.MAX_VALUE; Integer key = new Integer(loadOnStartup); ArrayList list = (ArrayList) map.get(key); if (list == null) { list = new ArrayList(); map.put(key, list); } list.add(wrapper); } // Load the collected "load on startup" servlets Iterator keys = map.keySet().iterator(); while (keys.hasNext()) { Integer key = (Integer) keys.next(); ArrayList list = (ArrayList) map.get(key); Iterator wrappers = list.iterator(); while (wrappers.hasNext()) { Wrapper wrapper = (Wrapper) wrappers.next(); try { wrapper.load(); } catch (ServletException e) { getServletContext().log(sm.getString("standardWrapper.loadException", getName()), e); // NOTE: load errors (including a servlet that throws // UnavailableException from tht init() method) are NOT // fatal to application startup } } } }
From source file:com.if3games.chessonline.DroidFish.java
@Override public void setRemainingTime(int wTime, int bTime, int nextUpdate) { if (ctrl.getGameMode().clocksActive()) { whiteTitleText.setText(getString(R.string.white_square_character) + " " + timeToString(wTime)); blackTitleText.setText(getString(R.string.black_square_character) + " " + timeToString(bTime)); } else {// ww w . j ava 2 s.c o m TreeMap<String, String> headers = new TreeMap<String, String>(); ctrl.getHeaders(headers); whiteTitleText.setText(headers.get("White")); blackTitleText.setText(headers.get("Black")); } handlerTimer.removeCallbacks(r); if (nextUpdate > 0) handlerTimer.postDelayed(r, nextUpdate); }
From source file:fr.cirad.mgdb.exporting.individualoriented.DARwinExportHandler.java
@Override public void exportData(OutputStream outputStream, String sModule, Collection<File> individualExportFiles, boolean fDeleteSampleExportFilesOnExit, ProgressIndicator progress, DBCursor markerCursor, Map<Comparable, Comparable> markerSynonyms, Map<String, InputStream> readyToExportFiles) throws Exception { MongoTemplate mongoTemplate = MongoTemplateManager.get(sModule); GenotypingProject aProject = mongoTemplate.findOne( new Query(Criteria.where(GenotypingProject.FIELDNAME_PLOIDY_LEVEL).exists(true)), GenotypingProject.class); if (aProject == null) LOG.warn("Unable to find a project containing ploidy level information! Assuming ploidy level is 2."); int ploidy = aProject == null ? 2 : aProject.getPloidyLevel(); File warningFile = File.createTempFile("export_warnings_", ""); FileWriter warningFileWriter = new FileWriter(warningFile); int markerCount = markerCursor.count(); ZipOutputStream zos = new ZipOutputStream(outputStream); if (readyToExportFiles != null) for (String readyToExportFile : readyToExportFiles.keySet()) { zos.putNextEntry(new ZipEntry(readyToExportFile)); InputStream inputStream = readyToExportFiles.get(readyToExportFile); byte[] dataBlock = new byte[1024]; int count = inputStream.read(dataBlock, 0, 1024); while (count != -1) { zos.write(dataBlock, 0, count); count = inputStream.read(dataBlock, 0, 1024); }/* w w w . ja va2 s . c om*/ } String exportName = sModule + "_" + markerCount + "variants_" + individualExportFiles.size() + "individuals"; StringBuffer donFileContents = new StringBuffer( "@DARwin 5.0 - DON -" + LINE_SEPARATOR + individualExportFiles.size() + "\t" + 1 + LINE_SEPARATOR + "N" + "\t" + "individual" + LINE_SEPARATOR); int count = 0; String missingGenotype = ""; for (int j = 0; j < ploidy; j++) missingGenotype += "\tN"; zos.putNextEntry(new ZipEntry(exportName + ".var")); zos.write(("@DARwin 5.0 - ALLELIC - " + ploidy + LINE_SEPARATOR + individualExportFiles.size() + "\t" + markerCount * ploidy + LINE_SEPARATOR + "N").getBytes()); DBCursor markerCursorCopy = markerCursor.copy(); // dunno how expensive this is, but seems safer than keeping all IDs in memory at any time short nProgress = 0, nPreviousProgress = 0; int avgObjSize = (Integer) mongoTemplate .getCollection(mongoTemplate.getCollectionName(VariantRunData.class)).getStats().get("avgObjSize"); int nChunkSize = nMaxChunkSizeInMb * 1024 * 1024 / avgObjSize; markerCursorCopy.batchSize(nChunkSize); int nMarkerIndex = 0; while (markerCursorCopy.hasNext()) { DBObject exportVariant = markerCursorCopy.next(); Comparable markerId = (Comparable) exportVariant.get("_id"); if (markerSynonyms != null) { Comparable syn = markerSynonyms.get(markerId); if (syn != null) markerId = syn; } for (int j = 0; j < ploidy; j++) zos.write(("\t" + markerId).getBytes()); } TreeMap<Integer, Comparable> problematicMarkerIndexToNameMap = new TreeMap<Integer, Comparable>(); ArrayList<String> distinctAlleles = new ArrayList<String>(); // the index of each allele will be used as its code int i = 0; for (File f : individualExportFiles) { BufferedReader in = new BufferedReader(new FileReader(f)); try { String individualId, line = in.readLine(); // read sample id if (line != null) individualId = line; else throw new Exception("Unable to read first line of temp export file " + f.getName()); donFileContents.append(++count + "\t" + individualId + LINE_SEPARATOR); zos.write((LINE_SEPARATOR + count).getBytes()); nMarkerIndex = 0; while ((line = in.readLine()) != null) { List<String> genotypes = MgdbDao.split(line, "|"); HashMap<Object, Integer> genotypeCounts = new HashMap<Object, Integer>(); // will help us to keep track of missing genotypes int highestGenotypeCount = 0; String mostFrequentGenotype = null; for (String genotype : genotypes) { if (genotype.length() == 0) continue; /* skip missing genotypes */ int gtCount = 1 + MgdbDao.getCountForKey(genotypeCounts, genotype); if (gtCount > highestGenotypeCount) { highestGenotypeCount = gtCount; mostFrequentGenotype = genotype; } genotypeCounts.put(genotype, gtCount); } if (genotypeCounts.size() > 1) { warningFileWriter.write("- Dissimilar genotypes found for variant __" + nMarkerIndex + "__, individual " + individualId + ". Exporting most frequent: " + mostFrequentGenotype + "\n"); problematicMarkerIndexToNameMap.put(nMarkerIndex, ""); } String codedGenotype = ""; if (mostFrequentGenotype != null) for (String allele : mostFrequentGenotype.split(" ")) { if (!distinctAlleles.contains(allele)) distinctAlleles.add(allele); codedGenotype += "\t" + distinctAlleles.indexOf(allele); } else codedGenotype = missingGenotype.replaceAll("N", "-1"); // missing data is coded as -1 zos.write(codedGenotype.getBytes()); nMarkerIndex++; } } catch (Exception e) { LOG.error("Error exporting data", e); progress.setError("Error exporting data: " + e.getClass().getSimpleName() + (e.getMessage() != null ? " - " + e.getMessage() : "")); return; } finally { in.close(); } if (progress.hasAborted()) return; nProgress = (short) (++i * 100 / individualExportFiles.size()); if (nProgress > nPreviousProgress) { // LOG.debug("============= doDARwinExport (" + i + "): " + nProgress + "% ============="); progress.setCurrentStepProgress(nProgress); nPreviousProgress = nProgress; } if (!f.delete()) { f.deleteOnExit(); LOG.info("Unable to delete tmp export file " + f.getAbsolutePath()); } } zos.putNextEntry(new ZipEntry(exportName + ".don")); zos.write(donFileContents.toString().getBytes()); // now read variant names for those that induced warnings nMarkerIndex = 0; markerCursor.batchSize(nChunkSize); while (markerCursor.hasNext()) { DBObject exportVariant = markerCursor.next(); if (problematicMarkerIndexToNameMap.containsKey(nMarkerIndex)) { Comparable markerId = (Comparable) exportVariant.get("_id"); if (markerSynonyms != null) { Comparable syn = markerSynonyms.get(markerId); if (syn != null) markerId = syn; } for (int j = 0; j < ploidy; j++) zos.write(("\t" + markerId).getBytes()); problematicMarkerIndexToNameMap.put(nMarkerIndex, markerId); } } warningFileWriter.close(); if (warningFile.length() > 0) { zos.putNextEntry(new ZipEntry(exportName + "-REMARKS.txt")); int nWarningCount = 0; BufferedReader in = new BufferedReader(new FileReader(warningFile)); String sLine; while ((sLine = in.readLine()) != null) { for (Integer aMarkerIndex : problematicMarkerIndexToNameMap.keySet()) sLine = sLine.replaceAll("__" + aMarkerIndex + "__", problematicMarkerIndexToNameMap.get(aMarkerIndex).toString()); zos.write((sLine + "\n").getBytes()); in.readLine(); nWarningCount++; } LOG.info("Number of Warnings for export (" + exportName + "): " + nWarningCount); in.close(); } warningFile.delete(); zos.close(); progress.setCurrentStepProgress((short) 100); }
From source file:com.maxl.java.aips2sqlite.RealExpertInfo.java
public void process() { // Get stop words first getStopWords();// w w w . j a v a 2 s .c o m // Extract EPha SwissmedicNo5 to ATC map extractSwissmedicNo5ToAtcMap(); // Extract package information (this is the heavy-duty bit) extractPackageInfo(); // Extract Swiss DRG information extractSwissDRGInfo(); try { // Load CSS file: used only for self-contained xml files String amiko_style_v1_str = FileOps.readCSSfromFile(Constants.FILE_STYLE_CSS_BASE + "v1.css"); // Create error report file ParseReport parse_errors = null; if (CmlOptions.GENERATE_REPORTS == true) { parse_errors = new ParseReport(Constants.FILE_PARSE_REPORT, CmlOptions.DB_LANGUAGE, "html"); if (CmlOptions.DB_LANGUAGE.equals("de")) parse_errors.addHtmlHeader("Schweizer Arzneimittel-Kompendium", Constants.FI_DB_VERSION); else if (CmlOptions.DB_LANGUAGE.equals("fr")) parse_errors.addHtmlHeader("Compendium des Mdicaments Suisse", Constants.FI_DB_VERSION); } // Create indications report file BufferedWriter bw_indications = null; Map<String, String> tm_indications = new TreeMap<String, String>(); if (CmlOptions.INDICATIONS_REPORT == true) { ParseReport indications_report = new ParseReport(Constants.FILE_INDICATIONS_REPORT, CmlOptions.DB_LANGUAGE, "txt"); bw_indications = indications_report.getBWriter(); } /* * Add pseudo Fachinfos to SQLite database */ int tot_pseudo_counter = 0; if (CmlOptions.ADD_PSEUDO_FI == true) { PseudoExpertInfo pseudo_fi = new PseudoExpertInfo(m_sql_db, CmlOptions.DB_LANGUAGE, m_map_products); // Process tot_pseudo_counter = pseudo_fi.process(); System.out.println(""); } /* * Add real Fachinfos to SQLite database */ // Initialize counters for different languages int med_counter = 0; int tot_med_counter = 0; int missing_regnr_str = 0; int missing_pack_info = 0; int missing_atc_code = 0; int errors = 0; String fi_complete_xml = ""; // First pass is always with DB_LANGUAGE set to German! (most complete information) // The file dumped in ./reports is fed to AllDown.java to generate a multilingual ATC code / ATC class file, e.g. German - French Set<String> atccode_set = new TreeSet<String>(); // Treemap for owner error report (sorted by key) TreeMap<String, ArrayList<String>> tm_owner_error = new TreeMap<String, ArrayList<String>>(); HtmlUtils html_utils = null; System.out.println("Processing real Fachinfos..."); for (MedicalInformations.MedicalInformation m : m_med_list) { // --> Read FACHINFOS! <-- if (m.getLang().equals(CmlOptions.DB_LANGUAGE) && m.getType().equals("fi")) { // Database contains less than 5000 medis - this is a safe upperbound! if (tot_med_counter < 5000) { // Trim titles of leading and trailing spaces m.setTitle(m.getTitle().trim()); // Extract section titles and section ids MedicalInformations.MedicalInformation.Sections med_sections = m.getSections(); List<MedicalInformations.MedicalInformation.Sections.Section> med_section_list = med_sections .getSection(); String ids_str = ""; String titles_str = ""; for (MedicalInformations.MedicalInformation.Sections.Section s : med_section_list) { ids_str += (s.getId() + ","); titles_str += (s.getTitle() + ";"); } Document doc = Jsoup.parse(m.getContent()); doc.outputSettings().escapeMode(EscapeMode.xhtml); html_utils = new HtmlUtils(m.getContent()); html_utils.setLanguage(CmlOptions.DB_LANGUAGE); html_utils.clean(); // Extract registration number (swissmedic no5) String regnr_str = ""; if (CmlOptions.DB_LANGUAGE.equals("de")) regnr_str = html_utils.extractRegNrDE(m.getTitle()); else if (CmlOptions.DB_LANGUAGE.equals("fr")) regnr_str = html_utils.extractRegNrFR(m.getTitle()); // Pattern matcher for regnr command line option, (?s) searches across multiple lines Pattern regnr_pattern = Pattern.compile("(?s).*\\b" + CmlOptions.OPT_MED_REGNR); if (m.getTitle().toLowerCase().startsWith(CmlOptions.OPT_MED_TITLE.toLowerCase()) && regnr_pattern.matcher(regnr_str).find() && m.getAuthHolder().toLowerCase() .startsWith(CmlOptions.OPT_MED_OWNER.toLowerCase())) { System.out.println(tot_med_counter + " - " + m.getTitle() + ": " + regnr_str); if (regnr_str.isEmpty()) { errors++; if (CmlOptions.GENERATE_REPORTS == true) { parse_errors.append("<p style=\"color:#ff0099\">ERROR " + errors + ": reg. nr. could not be parsed in AIPS.xml (swissmedic) - " + m.getTitle() + " (" + regnr_str + ")</p>"); // Add to owner errors ArrayList<String> error = tm_owner_error.get(m.getAuthHolder()); if (error == null) error = new ArrayList<String>(); error.add(m.getTitle() + ";regnr"); tm_owner_error.put(m.getAuthHolder(), error); } missing_regnr_str++; regnr_str = ""; } // Associate ATC classes and subclasses (atc_map) String atc_class_str = ""; String atc_description_str = ""; // This bit is necessary because the ATC Code in the AIPS DB is broken sometimes String atc_code_str = ""; boolean atc_error_found = false; // Use EPha ATC Codes, AIPS is fallback solution String authNrs = m.getAuthNrs(); if (authNrs != null) { // Deal with multi-swissmedic no5 case String regnrs[] = authNrs.split(","); // Use set to avoid duplicate ATC codes Set<String> regnrs_set = new LinkedHashSet<>(); // Loop through EPha ATC codes for (String r : regnrs) { regnrs_set.add(m_smn5_atc_map.get(r.trim())); } // Iterate through set and format nicely for (String r : regnrs_set) { if (atc_code_str == null || atc_code_str.isEmpty()) atc_code_str = r; else atc_code_str += "," + r; } } else atc_error_found = true; // Notify any other problem with the EPha ATC codes if (atc_code_str == null || atc_code_str.isEmpty()) atc_error_found = true; // Fallback solution if (atc_error_found == true) { if (m.getAtcCode() != null && !m.getAtcCode().equals("n.a.") && m.getAtcCode().length() > 1) { atc_code_str = m.getAtcCode(); atc_code_str = atc_code_str.replaceAll("–", "("); atc_code_str = atc_code_str.replaceAll("Code", "").replaceAll("ATC", "") .replaceAll(" ", "").replaceAll("\\(.*", "").replaceAll("/", ",") .replaceAll("[^A-Za-z0-9,]", ""); if (atc_code_str.charAt(1) == 'O') { // E.g. Ascosal Brausetabletten atc_code_str = atc_code_str.substring(0, 1) + '0' + atc_code_str.substring(2); } if (atc_code_str.length() > 7) { if (atc_code_str.charAt(7) != ',' || atc_code_str.length() != 15) atc_code_str = atc_code_str.substring(0, 7); } } else { // Work backwards using m_atc_map and m.getSubstances() String substances = m.getSubstances(); if (substances != null) { if (m_atc_map.containsValue(substances)) { for (Map.Entry<String, String> entry : m_atc_map.entrySet()) { if (entry.getValue().equals(substances)) { atc_code_str = entry.getKey(); } } } } } atc_error_found = false; } // Now let's clean the m.getSubstances() String substances = m.getSubstances(); if ((substances == null || substances.length() < 3) && atc_code_str != null) { substances = m_atc_map.get(atc_code_str); } // Set clean substances m.setSubstances(substances); // Set clean ATC Code m.setAtcCode(atc_code_str); // System.out.println("ATC -> " + atc_code_str + ": " + substances); if (atc_code_str != null) { // \\s -> whitespace character, short for [ \t\n\x0b\r\f] // atc_code_str = atc_code_str.replaceAll("\\s",""); // Take "leave" of the tree (most precise classification) String a = m_atc_map.get(atc_code_str); if (a != null) { atc_description_str = a; atccode_set.add(atc_code_str + ": " + a); } else { // Case: ATC1,ATC2 if (atc_code_str.length() == 15) { String[] codes = atc_code_str.split(","); if (codes.length > 1) { String a1 = m_atc_map.get(codes[0]); if (a1 == null) { atc_error_found = true; a1 = "k.A."; } String a2 = m_atc_map.get(codes[1]); if (a2 == null) { atc_error_found = true; a2 = "k.A."; } atc_description_str = a1 + "," + a2; } } else if (m.getSubstances() != null) { // Fallback in case nothing else works atc_description_str = m.getSubstances(); // Work backwards using m_atc_map and m.getSubstances(), change ATC code if (atc_description_str != null) { if (m_atc_map.containsValue(atc_description_str)) { for (Map.Entry<String, String> entry : m_atc_map.entrySet()) { if (entry.getValue().equals(atc_description_str)) { m.setAtcCode(entry.getKey()); } } } } } else { atc_error_found = true; if (CmlOptions.DB_LANGUAGE.equals("de")) atc_description_str = "k.A."; else if (CmlOptions.DB_LANGUAGE.equals("fr")) atc_description_str = "n.s."; } } // Read out only two levels (L1, L3, L4, L5) for (int i = 1; i < 6; i++) { if (i != 2) { String atc_key = ""; if (i <= atc_code_str.length()) atc_key = atc_code_str.substring(0, i); char sep = (i >= 4) ? '#' : ';'; // #-separator between L4 and L5 if (atc_key != null) { String c = m_atc_map.get(atc_key); if (c != null) { atccode_set.add(atc_key + ": " + c); atc_class_str += (c + sep); } else { atc_class_str += sep; } } else { atc_class_str += sep; } } } // System.out.println("atc class = " + atc_class_str); // If DRG medication, add to atc_description_str ArrayList<String> drg = m_swiss_drg_info.get(atc_code_str); if (drg != null) { atc_description_str += (";DRG"); } } if (atc_error_found) { errors++; if (CmlOptions.GENERATE_REPORTS) { parse_errors.append("<p style=\"color:#0000bb\">ERROR " + errors + ": Broken or missing ATC-Code-Tag in AIPS.xml (Swissmedic) or ATC index (Wido) - " + m.getTitle() + " (" + regnr_str + ")</p>"); // Add to owner errors ArrayList<String> error = tm_owner_error.get(m.getAuthHolder()); if (error == null) error = new ArrayList<String>(); error.add(m.getTitle() + ";atccode"); tm_owner_error.put(m.getAuthHolder(), error); } System.err.println(">> ERROR: " + tot_med_counter + " - no ATC-Code found in the XML-Tag \"atcCode\" - (" + regnr_str + ") " + m.getTitle()); missing_atc_code++; } // Additional info stored in add_info_map String add_info_str = ";"; List<String> rnr_list = Arrays.asList(regnr_str.split("\\s*, \\s*")); if (rnr_list.size() > 0) add_info_str = m_add_info_map.get(rnr_list.get(0)); // Sanitize html String html_sanitized = ""; // First check for bad boys (version=1! but actually version>1!) if (!m.getVersion().equals("1") || m.getContent().substring(0, 20).contains("xml")) { for (int i = 1; i < 22; ++i) { html_sanitized += html_utils.sanitizeSection(i, m.getTitle(), m.getAuthHolder(), CmlOptions.DB_LANGUAGE); } html_sanitized = "<div id=\"monographie\">" + html_sanitized + "</div>"; } else { html_sanitized = m.getContent(); } // Add author number html_sanitized = html_sanitized.replaceAll("<div id=\"monographie\">", "<div id=\"monographie\" name=\"" + m.getAuthNrs() + "\">"); // Add Footer, timestamp in RFC822 format DateFormat dateFormat = new SimpleDateFormat("EEE', 'dd' 'MMM' 'yyyy' 'HH:mm:ss' 'Z", Locale.getDefault()); Date date = new Date(); String footer_str = "<p class=\"footer\">Auto-generated by <a href=\"https://github.com/zdavatz/aips2sqlite\">aips2sqlite</a> on " + dateFormat.format(date) + "</p>"; // html_sanitized += footer_str; html_sanitized = html_sanitized.replaceAll("</div>$", footer_str + "</div>"); // Extract section indications String section_indications = ""; if (CmlOptions.DB_LANGUAGE.equals("de")) { String sstr1 = "Indikationen/Anwendungsmglichkeiten"; String sstr2 = "Dosierung/Anwendung"; if (html_sanitized.contains(sstr1) && html_sanitized.contains(sstr2)) { int idx1 = html_sanitized.indexOf(sstr1) + sstr1.length(); int idx2 = html_sanitized.substring(idx1, html_sanitized.length()) .indexOf(sstr2); try { section_indications = html_sanitized.substring(idx1, idx1 + idx2); } catch (StringIndexOutOfBoundsException e) { e.printStackTrace(); } } } else if (CmlOptions.DB_LANGUAGE.equals("fr")) { String sstr1 = "Indications/Possibilits demploi"; String sstr2 = "Posologie/Mode demploi"; html_sanitized = html_sanitized.replaceAll("Indications/Possibilits d'emploi", sstr1); html_sanitized = html_sanitized.replaceAll("Posologie/Mode d'emploi", sstr2); html_sanitized = html_sanitized.replaceAll("Indications/possibilits demploi", sstr1); html_sanitized = html_sanitized.replaceAll("Posologie/mode demploi", sstr2); if (html_sanitized.contains(sstr1) && html_sanitized.contains(sstr2)) { int idx1 = html_sanitized.indexOf(sstr1) + sstr1.length(); int idx2 = html_sanitized.substring(idx1, html_sanitized.length()) .indexOf(sstr2); try { section_indications = html_sanitized.substring(idx1, idx1 + idx2); } catch (StringIndexOutOfBoundsException e) { e.printStackTrace(); } } } // Remove all p's, div's, span's and sup's section_indications = section_indications.replaceAll("\\<p.*?\\>", "") .replaceAll("</p>", ""); section_indications = section_indications.replaceAll("\\<div.*?\\>", "") .replaceAll("</div>", ""); section_indications = section_indications.replaceAll("\\<span.*?\\>", "") .replaceAll("</span>", ""); section_indications = section_indications.replaceAll("\\<sup.*?\\>", "") .replaceAll("</sup>", ""); // System.out.println(section_indications); if (CmlOptions.DB_LANGUAGE.equals("fr")) { // Remove apostrophes section_indications = section_indications.replaceAll("l'", "") .replaceAll("d'", ""); section_indications = section_indications.replaceAll("l", "").replaceAll("d", ""); } // Remove all URLs section_indications = section_indications.replaceAll( "\\b(http|ftp|file)://[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]", ""); // Remove list of type a) b) c) ... 1) 2) ... section_indications = section_indications.replaceAll("^\\w\\)", ""); // Remove numbers, commas, semicolons, parentheses, etc. section_indications = section_indications.replaceAll("[^A-Za-z\\xC0-\\xFF- ]", ""); // Generate long list of keywords LinkedList<String> wordsAsList = new LinkedList<String>( Arrays.asList(section_indications.split("\\s+"))); // Remove stop words Iterator<String> wordIterator = wordsAsList.iterator(); while (wordIterator.hasNext()) { // Note: This assumes there are no null entries in the list and all stopwords are stored in lower case String word = wordIterator.next().trim().toLowerCase(); if (word.length() < 3 || m.getTitle().toLowerCase().contains(word) || m_stop_words_hash.contains(word)) wordIterator.remove(); } section_indications = ""; for (String w : wordsAsList) { // Remove any leading dash or hyphen if (w.startsWith("-")) w = w.substring(1); section_indications += (w + ";"); if (CmlOptions.INDICATIONS_REPORT == true) { // Add to map (key->value), word = key, value = how many times used // Is word w already stored in treemap? String t_str = tm_indications.get(w); if (t_str == null) { t_str = m.getTitle(); tm_indications.put(w, t_str); } else { t_str += (", " + m.getTitle()); tm_indications.put(w, t_str); } } } /* * Update section "Packungen", generate packungen string for shopping cart, and extract therapeutisches index */ List<String> mTyIndex_list = new ArrayList<String>(); m_list_of_packages.clear(); m_list_of_eancodes.clear(); String mContent_str = updateSectionPackungen(m.getTitle(), m.getAtcCode(), m_package_info, regnr_str, html_sanitized, mTyIndex_list); m.setContent(mContent_str); // Check if mPackSection_str is empty AND command line option PLAIN is not active if (CmlOptions.PLAIN == false && m_pack_info_str.isEmpty()) { errors++; if (CmlOptions.GENERATE_REPORTS) { parse_errors.append("<p style=\"color:#bb0000\">ERROR " + errors + ": SwissmedicNo5 not found in Packungen.xls (Swissmedic) - " + m.getTitle() + " (" + regnr_str + ")</p>"); // Add to owner errors ArrayList<String> error = tm_owner_error.get(m.getAuthHolder()); if (error == null) error = new ArrayList<String>(); error.add(m.getTitle() + ";swissmedic5"); tm_owner_error.put(m.getAuthHolder(), error); } System.err.println(">> ERROR: " + tot_med_counter + " - SwissmedicNo5 not found in Swissmedic Packungen.xls - (" + regnr_str + ") " + m.getTitle()); missing_pack_info++; } // Fix problem with wrong div class in original Swissmedic file if (CmlOptions.DB_LANGUAGE.equals("de")) { m.setStyle(m.getStyle().replaceAll("untertitel", "untertitle")); m.setStyle(m.getStyle().replaceAll("untertitel1", "untertitle1")); } // Correct formatting error introduced by Swissmedic m.setAuthHolder(m.getAuthHolder().replaceAll("&", "&")); // Check if substances str has a '$a' and change it to '&alpha' if (m.getSubstances() != null) m.setSubstances(m.getSubstances().replaceAll("\\$a", "α")); if (CmlOptions.XML_FILE == true) { if (!regnr_str.isEmpty()) { // Generate and add hash code String html_str_no_timestamp = mContent_str .replaceAll("<p class=\"footer\">.*?</p>", ""); String hash_code = html_utils.calcHashCode(html_str_no_timestamp); // Add header to html file mContent_str = mContent_str.replaceAll("<head>", "<head>" + "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" name=\"fi_" + hash_code + "\"/>" + "<style>" + amiko_style_v1_str + "</style>"); // Note: the following line is not necessary! // m.setContent(mContent_str); // Add header to xml file String xml_str = html_utils.convertHtmlToXml("fi", m.getTitle(), mContent_str, regnr_str); xml_str = html_utils.addHeaderToXml("singlefi", xml_str); fi_complete_xml += (xml_str + "\n"); // Write to html and xml files to disk String name = m.getTitle(); // Replace all "Sonderzeichen" name = name.replaceAll("[^a-zA-Z0-9]+", "_"); if (CmlOptions.DB_LANGUAGE.equals("de")) { FileOps.writeToFile(mContent_str, Constants.FI_FILE_XML_BASE + "fi_de_html/", name + "_fi_de.html"); FileOps.writeToFile(xml_str, Constants.FI_FILE_XML_BASE + "fi_de_xml/", name + "_fi_de.xml"); } else if (CmlOptions.DB_LANGUAGE.equals("fr")) { FileOps.writeToFile(mContent_str, Constants.FI_FILE_XML_BASE + "fi_fr_html/", name + "_fi_fr.html"); FileOps.writeToFile(xml_str, Constants.FI_FILE_XML_BASE + "fi_fr_xml/", name + "_fi_fr.xml"); } } } int customer_id = 0; // Is the customer paying? If yes add customer id // str1.toLowerCase().contains(str2.toLowerCase()) if (m.getAuthHolder().toLowerCase().contains("desitin")) customer_id = 1; /* / HERE GO THE OTHER PAYING CUSTOMERS (increment customer_id respectively) */ // Extract (O)riginal / (G)enerika info String orggen_str = ""; if (add_info_str != null) { List<String> ai_list = Arrays.asList(add_info_str.split("\\s*;\\s*")); if (ai_list != null) { if (!ai_list.get(0).isEmpty()) orggen_str = ai_list.get(0); } } // @maxl: 25.04.2015 -> set orggen_str to nil (we are using add_info_str for group names now...) orggen_str = ""; /* * Add medis, titles and ids to database */ String packages_str = ""; for (String s : m_list_of_packages) packages_str += s; String eancodes_str = ""; for (String e : m_list_of_eancodes) eancodes_str += (e + ", "); if (!eancodes_str.isEmpty() && eancodes_str.length() > 2) eancodes_str = eancodes_str.substring(0, eancodes_str.length() - 2); m_sql_db.addExpertDB(m, packages_str, regnr_str, ids_str, titles_str, atc_description_str, atc_class_str, m_pack_info_str, orggen_str, customer_id, mTyIndex_list, section_indications); m_sql_db.addProductDB(m, packages_str, eancodes_str, m_pack_info_str); med_counter++; } } tot_med_counter++; } } System.out.println(); System.out.println("--------------------------------------------"); System.out.println("Total number of real Fachinfos: " + m_med_list.size()); System.out.println("Number of FI with package information: " + tot_med_counter); System.out.println("Number of FI in generated database: " + med_counter); System.out.println("Number of errors in db: " + errors); System.out.println("Number of missing reg. nr. (min): " + missing_regnr_str); System.out.println("Number of missing pack info: " + missing_pack_info); System.out.println("Number of missing atc codes: " + missing_atc_code); System.out.println("--------------------------------------------"); System.out.println("Total number of pseudo Fachinfos: " + tot_pseudo_counter); System.out.println("--------------------------------------------"); if (CmlOptions.XML_FILE == true) { fi_complete_xml = html_utils.addHeaderToXml("kompendium", fi_complete_xml); // Write kompendium xml file to disk if (CmlOptions.DB_LANGUAGE.equals("de")) { FileOps.writeToFile(fi_complete_xml, Constants.FI_FILE_XML_BASE, "fi_de.xml"); if (CmlOptions.ZIP_BIG_FILES) FileOps.zipToFile(Constants.FI_FILE_XML_BASE, "fi_de.xml"); } else if (CmlOptions.DB_LANGUAGE.equals("fr")) { FileOps.writeToFile(fi_complete_xml, Constants.FI_FILE_XML_BASE, "fi_fr.xml"); if (CmlOptions.ZIP_BIG_FILES) FileOps.zipToFile(Constants.FI_FILE_XML_BASE, "fi_fr.xml"); } // Copy stylesheet file to ./fis/ folders try { File src = new File(Constants.FILE_STYLE_CSS_BASE + "v1.css"); File dst_de = new File(Constants.FI_FILE_XML_BASE + "fi_de_html/"); File dst_fr = new File(Constants.FI_FILE_XML_BASE + "fi_fr_html/"); if (src.exists()) { if (dst_de.exists()) FileUtils.copyFileToDirectory(src, dst_de); if (dst_fr.exists()) FileUtils.copyFileToDirectory(src, dst_fr); } } catch (IOException e) { // TODO: Unhandled! } } if (CmlOptions.GENERATE_REPORTS == true) { parse_errors.append("<br/>"); parse_errors .append("<p>Number of medications with package information: " + tot_med_counter + "</p>"); parse_errors.append("<p>Number of medications in generated database: " + med_counter + "</p>"); parse_errors.append("<p>Number of errors in database: " + errors + "</p>"); parse_errors.append("<p>Number of missing registration number: " + missing_regnr_str + "</p>"); parse_errors.append("<p>Number of missing package info: " + missing_pack_info + "</p>"); parse_errors.append("<p>Number of missing atc codes: " + missing_atc_code + "</p>"); parse_errors.append("<br/>"); // Write and close report file parse_errors.writeHtmlToFile(); parse_errors.getBWriter().close(); // Write owner error report to file ParseReport owner_errors = new ParseReport(Constants.FILE_OWNER_REPORT, CmlOptions.DB_LANGUAGE, "html"); String report_style_str = FileOps.readCSSfromFile(Constants.FILE_REPORT_CSS_BASE + ".css"); owner_errors.addStyleSheet(report_style_str); if (CmlOptions.DB_LANGUAGE.equals("de")) owner_errors.addHtmlHeader("Schweizer Arzneimittel-Kompendium", Constants.FI_DB_VERSION); else if (CmlOptions.DB_LANGUAGE.equals("fr")) owner_errors.addHtmlHeader("Compendium des Mdicaments Suisse", Constants.FI_DB_VERSION); owner_errors.append(owner_errors.treemapToHtmlTable(tm_owner_error)); owner_errors.writeHtmlToFile(); owner_errors.getBWriter().close(); // Dump to console... /* for (Map.Entry<String, ArrayList<String>> entry : tm_owner_error.entrySet()) { String author = entry.getKey(); ArrayList<String> list = entry.getValue(); for (String error : list) System.out.println(author + " -> " + error); } */ } if (CmlOptions.INDICATIONS_REPORT == true) { // Dump everything to file bw_indications.write("Total number of words: " + tm_indications.size() + "\n\n"); for (Map.Entry<String, String> entry : tm_indications.entrySet()) { String key = entry.getKey(); String value = entry.getValue(); bw_indications.write(key + " [" + value + "]\n"); } bw_indications.close(); } if (CmlOptions.DB_LANGUAGE.equals("de")) { // Dump set to file, currently we do this only for German File atccodes_file = new File("./output/atc_codes_used_set.txt"); if (!atccodes_file.exists()) { atccodes_file.getParentFile().mkdirs(); atccodes_file.createNewFile(); } FileWriter fwriter = new FileWriter(atccodes_file.getAbsoluteFile()); BufferedWriter bwriter = new BufferedWriter(fwriter); Iterator<String> set_iterator = atccode_set.iterator(); while (set_iterator.hasNext()) { bwriter.write(set_iterator.next() + "\n"); } bwriter.close(); } System.out.println(""); } catch (IOException e) { e.printStackTrace(); } }
From source file:fr.cirad.mgdb.exporting.markeroriented.GFFExportHandler.java
@Override public void exportData(OutputStream outputStream, String sModule, List<SampleId> sampleIDs, ProgressIndicator progress, DBCursor markerCursor, Map<Comparable, Comparable> markerSynonyms, int nMinimumGenotypeQuality, int nMinimumReadDepth, Map<String, InputStream> readyToExportFiles) throws Exception { MongoTemplate mongoTemplate = MongoTemplateManager.get(sModule); ZipOutputStream zos = new ZipOutputStream(outputStream); if (readyToExportFiles != null) for (String readyToExportFile : readyToExportFiles.keySet()) { zos.putNextEntry(new ZipEntry(readyToExportFile)); InputStream inputStream = readyToExportFiles.get(readyToExportFile); byte[] dataBlock = new byte[1024]; int count = inputStream.read(dataBlock, 0, 1024); while (count != -1) { zos.write(dataBlock, 0, count); count = inputStream.read(dataBlock, 0, 1024); }//from w ww.ja va2s . c om } File warningFile = File.createTempFile("export_warnings_", ""); FileWriter warningFileWriter = new FileWriter(warningFile); int markerCount = markerCursor.count(); List<Individual> individuals = getIndividualsFromSamples(sModule, sampleIDs); ArrayList<String> individualList = new ArrayList<String>(); for (int i = 0; i < sampleIDs.size(); i++) { Individual individual = individuals.get(i); if (!individualList.contains(individual.getId())) { individualList.add(individual.getId()); } } String exportName = sModule + "_" + markerCount + "variants_" + individualList.size() + "individuals"; zos.putNextEntry(new ZipEntry(exportName + ".gff3")); String header = "##gff-version 3" + LINE_SEPARATOR; zos.write(header.getBytes()); TreeMap<String, String> typeToOntology = new TreeMap<String, String>(); typeToOntology.put(Type.SNP.toString(), "SO:0000694"); typeToOntology.put(Type.INDEL.toString(), "SO:1000032"); typeToOntology.put(Type.MIXED.toString(), "SO:0001059"); typeToOntology.put(Type.SYMBOLIC.toString(), "SO:0000109"); typeToOntology.put(Type.MNP.toString(), "SO:0001059"); int avgObjSize = (Integer) mongoTemplate .getCollection(mongoTemplate.getCollectionName(VariantRunData.class)).getStats().get("avgObjSize"); int nChunkSize = nMaxChunkSizeInMb * 1024 * 1024 / avgObjSize; short nProgress = 0, nPreviousProgress = 0; long nLoadedMarkerCount = 0; while (markerCursor.hasNext()) { int nLoadedMarkerCountInLoop = 0; Map<Comparable, String> markerChromosomalPositions = new LinkedHashMap<Comparable, String>(); boolean fStartingNewChunk = true; markerCursor.batchSize(nChunkSize); while (markerCursor.hasNext() && (fStartingNewChunk || nLoadedMarkerCountInLoop % nChunkSize != 0)) { DBObject exportVariant = markerCursor.next(); DBObject refPos = (DBObject) exportVariant.get(VariantData.FIELDNAME_REFERENCE_POSITION); markerChromosomalPositions.put((Comparable) exportVariant.get("_id"), refPos.get(ReferencePosition.FIELDNAME_SEQUENCE) + ":" + refPos.get(ReferencePosition.FIELDNAME_START_SITE)); nLoadedMarkerCountInLoop++; fStartingNewChunk = false; } List<Comparable> currentMarkers = new ArrayList<Comparable>(markerChromosomalPositions.keySet()); LinkedHashMap<VariantData, Collection<VariantRunData>> variantsAndRuns = MgdbDao.getSampleGenotypes( mongoTemplate, sampleIDs, currentMarkers, true, null /*new Sort(VariantData.FIELDNAME_REFERENCE_POSITION + "." + ChromosomalPosition.FIELDNAME_SEQUENCE).and(new Sort(VariantData.FIELDNAME_REFERENCE_POSITION + "." + ChromosomalPosition.FIELDNAME_START_SITE))*/); // query mongo db for matching genotypes for (VariantData variant : variantsAndRuns.keySet()) // read data and write results into temporary files (one per sample) { Comparable variantId = variant.getId(); List<String> variantDataOrigin = new ArrayList<String>(); Map<String, Integer> gqValueForSampleId = new LinkedHashMap<String, Integer>(); Map<String, Integer> dpValueForSampleId = new LinkedHashMap<String, Integer>(); Map<String, List<String>> individualGenotypes = new LinkedHashMap<String, List<String>>(); List<String> chromAndPos = Helper.split(markerChromosomalPositions.get(variantId), ":"); if (chromAndPos.size() == 0) LOG.warn("Chromosomal position not found for marker " + variantId); // LOG.debug(marker + "\t" + (chromAndPos.length == 0 ? "0" : chromAndPos[0]) + "\t" + 0 + "\t" + (chromAndPos.length == 0 ? 0l : Long.parseLong(chromAndPos[1])) + LINE_SEPARATOR); if (markerSynonyms != null) { Comparable syn = markerSynonyms.get(variantId); if (syn != null) variantId = syn; } Collection<VariantRunData> runs = variantsAndRuns.get(variant); if (runs != null) for (VariantRunData run : runs) for (Integer sampleIndex : run.getSampleGenotypes().keySet()) { SampleGenotype sampleGenotype = run.getSampleGenotypes().get(sampleIndex); String individualId = individuals .get(sampleIDs.indexOf(new SampleId(run.getId().getProjectId(), sampleIndex))) .getId(); Integer gq = null; try { gq = (Integer) sampleGenotype.getAdditionalInfo().get(VariantData.GT_FIELD_GQ); } catch (Exception ignored) { } if (gq != null && gq < nMinimumGenotypeQuality) continue; Integer dp = null; try { dp = (Integer) sampleGenotype.getAdditionalInfo().get(VariantData.GT_FIELD_DP); } catch (Exception ignored) { } if (dp != null && dp < nMinimumReadDepth) continue; String gtCode = sampleGenotype.getCode(); List<String> storedIndividualGenotypes = individualGenotypes.get(individualId); if (storedIndividualGenotypes == null) { storedIndividualGenotypes = new ArrayList<String>(); individualGenotypes.put(individualId, storedIndividualGenotypes); } storedIndividualGenotypes.add(gtCode); } zos.write((chromAndPos.get(0) + "\t" + StringUtils.join(variantDataOrigin, ";") /*source*/ + "\t" + typeToOntology.get(variant.getType()) + "\t" + Long.parseLong(chromAndPos.get(1)) + "\t" + Long.parseLong(chromAndPos.get(1)) + "\t" + "." + "\t" + "+" + "\t" + "." + "\t") .getBytes()); Comparable syn = markerSynonyms == null ? null : markerSynonyms.get(variant.getId()); zos.write(("ID=" + variant.getId() + ";" + (syn != null ? "Name=" + syn + ";" : "") + "alleles=" + StringUtils.join(variant.getKnownAlleleList(), "/") + ";" + "refallele=" + variant.getKnownAlleleList().get(0) + ";").getBytes()); for (int j = 0; j < individualList .size(); j++ /* we use this list because it has the proper ordering*/) { NumberFormat nf = NumberFormat.getInstance(Locale.US); nf.setMaximumFractionDigits(4); HashMap<String, Integer> compt1 = new HashMap<String, Integer>(); int highestGenotypeCount = 0; int sum = 0; String individualId = individualList.get(j); List<String> genotypes = individualGenotypes.get(individualId); HashMap<Object, Integer> genotypeCounts = new HashMap<Object, Integer>(); // will help us to keep track of missing genotypes String mostFrequentGenotype = null; if (genotypes != null) for (String genotype : genotypes) { if (genotype.length() == 0) continue; /* skip missing genotypes */ int count = 0; for (String t : variant.getAllelesFromGenotypeCode(genotype)) { for (String t1 : variant.getKnownAlleleList()) { if (t.equals(t1) && !(compt1.containsKey(t1))) { count++; compt1.put(t1, count); } else if (t.equals(t1) && compt1.containsKey(t1)) { if (compt1.get(t1) != 0) { count++; compt1.put(t1, count); } else compt1.put(t1, count); } else if (!(compt1.containsKey(t1))) { compt1.put(t1, 0); } } } for (int countValue : compt1.values()) { sum += countValue; } int gtCount = 1 + MgdbDao.getCountForKey(genotypeCounts, genotype); if (gtCount > highestGenotypeCount) { highestGenotypeCount = gtCount; mostFrequentGenotype = genotype; } genotypeCounts.put(genotype, gtCount); } List<String> alleles = mostFrequentGenotype == null ? new ArrayList<String>() : variant.getAllelesFromGenotypeCode(mostFrequentGenotype); if (alleles.size() != 0) { zos.write(("acounts=" + individualId + ":").getBytes()); for (String knowAllelesCompt : compt1.keySet()) { zos.write( (knowAllelesCompt + " " + nf.format(compt1.get(knowAllelesCompt) / (float) sum) + " " + compt1.get(knowAllelesCompt) + " ").getBytes()); } zos.write((alleles.size() + ";").getBytes()); } if (genotypeCounts.size() > 1) { Comparable sVariantId = markerSynonyms != null ? markerSynonyms.get(variant.getId()) : variant.getId(); warningFileWriter.write("- Dissimilar genotypes found for variant " + (sVariantId == null ? variant.getId() : sVariantId) + ", individual " + individualId + ". Exporting most frequent: " + StringUtils.join(alleles, ",") + "\n"); } } zos.write((LINE_SEPARATOR).getBytes()); } if (progress.hasAborted()) return; nLoadedMarkerCount += nLoadedMarkerCountInLoop; nProgress = (short) (nLoadedMarkerCount * 100 / markerCount); if (nProgress > nPreviousProgress) { // if (nProgress%5 == 0) // LOG.info("========================= exportData: " + nProgress + "% =========================" + (System.currentTimeMillis() - before)/1000 + "s"); progress.setCurrentStepProgress(nProgress); nPreviousProgress = nProgress; } } warningFileWriter.close(); if (warningFile.length() > 0) { zos.putNextEntry(new ZipEntry(exportName + "-REMARKS.txt")); int nWarningCount = 0; BufferedReader in = new BufferedReader(new FileReader(warningFile)); String sLine; while ((sLine = in.readLine()) != null) { zos.write((sLine + "\n").getBytes()); in.readLine(); nWarningCount++; } LOG.info("Number of Warnings for export (" + exportName + "): " + nWarningCount); in.close(); } warningFile.delete(); zos.close(); progress.setCurrentStepProgress((short) 100); }
From source file:ca.uhn.fhir.context.BaseRuntimeElementCompositeDefinition.java
@SuppressWarnings("unchecked") private void scanCompositeElementForChildren(Set<String> elementNames, TreeMap<Integer, BaseRuntimeDeclaredChildDefinition> theOrderToElementDef, TreeMap<Integer, BaseRuntimeDeclaredChildDefinition> theOrderToExtensionDef) { int baseElementOrder = 0; for (ScannedField next : myScannedFields) { if (next.isFirstFieldInNewClass()) { baseElementOrder = theOrderToElementDef.isEmpty() ? 0 : theOrderToElementDef.lastEntry().getKey() + 1; }//from w w w .j a v a 2 s . c o m Class<?> declaringClass = next.getField().getDeclaringClass(); Description descriptionAnnotation = ModelScanner.pullAnnotation(next.getField(), Description.class); TreeMap<Integer, BaseRuntimeDeclaredChildDefinition> orderMap = theOrderToElementDef; Extension extensionAttr = ModelScanner.pullAnnotation(next.getField(), Extension.class); if (extensionAttr != null) { orderMap = theOrderToExtensionDef; } Child childAnnotation = next.getChildAnnotation(); Field nextField = next.getField(); String elementName = childAnnotation.name(); int order = childAnnotation.order(); boolean childIsChoiceType = false; boolean orderIsReplaceParent = false; if (order == Child.REPLACE_PARENT) { if (extensionAttr != null) { for (Entry<Integer, BaseRuntimeDeclaredChildDefinition> nextEntry : orderMap.entrySet()) { BaseRuntimeDeclaredChildDefinition nextDef = nextEntry.getValue(); if (nextDef instanceof RuntimeChildDeclaredExtensionDefinition) { if (nextDef.getExtensionUrl().equals(extensionAttr.url())) { orderIsReplaceParent = true; order = nextEntry.getKey(); orderMap.remove(nextEntry.getKey()); elementNames.remove(elementName); break; } } } if (order == Child.REPLACE_PARENT) { throw new ConfigurationException("Field " + nextField.getName() + "' on target type " + declaringClass.getSimpleName() + " has order() of REPLACE_PARENT (" + Child.REPLACE_PARENT + ") but no parent element with extension URL " + extensionAttr.url() + " could be found on type " + nextField.getDeclaringClass().getSimpleName()); } } else { for (Entry<Integer, BaseRuntimeDeclaredChildDefinition> nextEntry : orderMap.entrySet()) { BaseRuntimeDeclaredChildDefinition nextDef = nextEntry.getValue(); if (elementName.equals(nextDef.getElementName())) { orderIsReplaceParent = true; order = nextEntry.getKey(); BaseRuntimeDeclaredChildDefinition existing = orderMap.remove(nextEntry.getKey()); elementNames.remove(elementName); /* * See #350 - If the original field (in the superclass) with the given name is a choice, then we need to make sure * that the field which replaces is a choice even if it's only a choice of one type - this is because the * element name when serialized still needs to reflect the datatype */ if (existing instanceof RuntimeChildChoiceDefinition) { childIsChoiceType = true; } break; } } if (order == Child.REPLACE_PARENT) { throw new ConfigurationException("Field " + nextField.getName() + "' on target type " + declaringClass.getSimpleName() + " has order() of REPLACE_PARENT (" + Child.REPLACE_PARENT + ") but no parent element with name " + elementName + " could be found on type " + nextField.getDeclaringClass().getSimpleName()); } } } if (order < 0 && order != Child.ORDER_UNKNOWN) { throw new ConfigurationException("Invalid order '" + order + "' on @Child for field '" + nextField.getName() + "' on target type: " + declaringClass); } if (order != Child.ORDER_UNKNOWN && !orderIsReplaceParent) { order = order + baseElementOrder; } // int min = childAnnotation.min(); // int max = childAnnotation.max(); /* * Anything that's marked as unknown is given a new ID that is <0 so that it doesn't conflict with any given IDs and can be figured out later */ if (order == Child.ORDER_UNKNOWN) { order = Integer.valueOf(0); while (orderMap.containsKey(order)) { order++; } } List<Class<? extends IBase>> choiceTypes = next.getChoiceTypes(); if (orderMap.containsKey(order)) { throw new ConfigurationException("Detected duplicate field order '" + childAnnotation.order() + "' for element named '" + elementName + "' in type '" + declaringClass.getCanonicalName() + "' - Already had: " + orderMap.get(order).getElementName()); } if (elementNames.contains(elementName)) { throw new ConfigurationException("Detected duplicate field name '" + elementName + "' in type '" + declaringClass.getCanonicalName() + "'"); } Class<?> nextElementType = next.getElementType(); BaseRuntimeDeclaredChildDefinition def; if (childAnnotation.name().equals("extension") && IBaseExtension.class.isAssignableFrom(nextElementType)) { def = new RuntimeChildExtension(nextField, childAnnotation.name(), childAnnotation, descriptionAnnotation); } else if (childAnnotation.name().equals("modifierExtension") && IBaseExtension.class.isAssignableFrom(nextElementType)) { def = new RuntimeChildExtension(nextField, childAnnotation.name(), childAnnotation, descriptionAnnotation); } else if (BaseContainedDt.class.isAssignableFrom(nextElementType) || (childAnnotation.name().equals("contained") && IBaseResource.class.isAssignableFrom(nextElementType))) { /* * Child is contained resources */ def = new RuntimeChildContainedResources(nextField, childAnnotation, descriptionAnnotation, elementName); } else if (IAnyResource.class.isAssignableFrom(nextElementType) || IResource.class.equals(nextElementType)) { /* * Child is a resource as a direct child, as in Bundle.entry.resource */ def = new RuntimeChildDirectResource(nextField, childAnnotation, descriptionAnnotation, elementName); } else { childIsChoiceType |= choiceTypes.size() > 1; if (childIsChoiceType && !BaseResourceReferenceDt.class.isAssignableFrom(nextElementType) && !IBaseReference.class.isAssignableFrom(nextElementType)) { def = new RuntimeChildChoiceDefinition(nextField, elementName, childAnnotation, descriptionAnnotation, choiceTypes); } else if (extensionAttr != null) { /* * Child is an extension */ Class<? extends IBase> et = (Class<? extends IBase>) nextElementType; Object binder = null; if (BoundCodeDt.class.isAssignableFrom(nextElementType) || IBoundCodeableConcept.class.isAssignableFrom(nextElementType)) { binder = ModelScanner.getBoundCodeBinder(nextField); } def = new RuntimeChildDeclaredExtensionDefinition(nextField, childAnnotation, descriptionAnnotation, extensionAttr, elementName, extensionAttr.url(), et, binder); if (IBaseEnumeration.class.isAssignableFrom(nextElementType)) { ((RuntimeChildDeclaredExtensionDefinition) def).setEnumerationType( ReflectionUtil.getGenericCollectionTypeOfFieldWithSecondOrderForList(nextField)); } } else if (BaseResourceReferenceDt.class.isAssignableFrom(nextElementType) || IBaseReference.class.isAssignableFrom(nextElementType)) { /* * Child is a resource reference */ List<Class<? extends IBaseResource>> refTypesList = new ArrayList<Class<? extends IBaseResource>>(); for (Class<? extends IElement> nextType : childAnnotation.type()) { if (IBaseReference.class.isAssignableFrom(nextType)) { refTypesList.add(myContext.getVersion().getVersion().isRi() ? IAnyResource.class : IResource.class); continue; } else if (IBaseResource.class.isAssignableFrom(nextType) == false) { throw new ConfigurationException("Field '" + nextField.getName() + "' in class '" + nextField.getDeclaringClass().getCanonicalName() + "' is of type " + BaseResourceReferenceDt.class + " but contains a non-resource type: " + nextType.getCanonicalName()); } refTypesList.add((Class<? extends IBaseResource>) nextType); } def = new RuntimeChildResourceDefinition(nextField, elementName, childAnnotation, descriptionAnnotation, refTypesList); } else if (IResourceBlock.class.isAssignableFrom(nextElementType) || IBaseBackboneElement.class.isAssignableFrom(nextElementType) || IBaseDatatypeElement.class.isAssignableFrom(nextElementType)) { /* * Child is a resource block (i.e. a sub-tag within a resource) TODO: do these have a better name according to HL7? */ Class<? extends IBase> blockDef = (Class<? extends IBase>) nextElementType; def = new RuntimeChildResourceBlockDefinition(myContext, nextField, childAnnotation, descriptionAnnotation, elementName, blockDef); } else if (IDatatype.class.equals(nextElementType) || IElement.class.equals(nextElementType) || "Type".equals(nextElementType.getSimpleName()) || IBaseDatatype.class.equals(nextElementType)) { def = new RuntimeChildAny(nextField, elementName, childAnnotation, descriptionAnnotation); } else if (IDatatype.class.isAssignableFrom(nextElementType) || IPrimitiveType.class.isAssignableFrom(nextElementType) || ICompositeType.class.isAssignableFrom(nextElementType) || IBaseDatatype.class.isAssignableFrom(nextElementType) || IBaseExtension.class.isAssignableFrom(nextElementType)) { Class<? extends IBase> nextDatatype = (Class<? extends IBase>) nextElementType; if (IPrimitiveType.class.isAssignableFrom(nextElementType)) { if (nextElementType.equals(BoundCodeDt.class)) { IValueSetEnumBinder<Enum<?>> binder = ModelScanner.getBoundCodeBinder(nextField); Class<? extends Enum<?>> enumType = ModelScanner .determineEnumTypeForBoundField(nextField); def = new RuntimeChildPrimitiveBoundCodeDatatypeDefinition(nextField, elementName, childAnnotation, descriptionAnnotation, nextDatatype, binder, enumType); } else if (IBaseEnumeration.class.isAssignableFrom(nextElementType)) { Class<? extends Enum<?>> binderType = ModelScanner .determineEnumTypeForBoundField(nextField); def = new RuntimeChildPrimitiveEnumerationDatatypeDefinition(nextField, elementName, childAnnotation, descriptionAnnotation, nextDatatype, binderType); } else { def = new RuntimeChildPrimitiveDatatypeDefinition(nextField, elementName, descriptionAnnotation, childAnnotation, nextDatatype); } } else { if (IBoundCodeableConcept.class.isAssignableFrom(nextElementType)) { IValueSetEnumBinder<Enum<?>> binder = ModelScanner.getBoundCodeBinder(nextField); Class<? extends Enum<?>> enumType = ModelScanner .determineEnumTypeForBoundField(nextField); def = new RuntimeChildCompositeBoundDatatypeDefinition(nextField, elementName, childAnnotation, descriptionAnnotation, nextDatatype, binder, enumType); } else if (BaseNarrativeDt.class.isAssignableFrom(nextElementType) || INarrative.class.isAssignableFrom(nextElementType)) { def = new RuntimeChildNarrativeDefinition(nextField, elementName, childAnnotation, descriptionAnnotation, nextDatatype); } else { def = new RuntimeChildCompositeDatatypeDefinition(nextField, elementName, childAnnotation, descriptionAnnotation, nextDatatype); } } } else { throw new ConfigurationException( "Field '" + elementName + "' in type '" + declaringClass.getCanonicalName() + "' is not a valid child type: " + nextElementType); } Binding bindingAnnotation = ModelScanner.pullAnnotation(nextField, Binding.class); if (bindingAnnotation != null) { if (isNotBlank(bindingAnnotation.valueSet())) { def.setBindingValueSet(bindingAnnotation.valueSet()); } } } orderMap.put(order, def); elementNames.add(elementName); } }
From source file:org.apache.hadoop.hive.metastore.MetaStoreDirectSql.java
/** Should be called with the list short enough to not trip up Oracle/etc. */ private List<Partition> getPartitionsFromPartitionIds(String dbName, String tblName, Boolean isView, List<Object> partIdList) throws MetaException { boolean doTrace = LOG.isDebugEnabled(); int idStringWidth = (int) Math.ceil(Math.log10(partIdList.size())) + 1; // 1 for comma int sbCapacity = partIdList.size() * idStringWidth; // Prepare StringBuilder for "PART_ID in (...)" to use in future queries. StringBuilder partSb = new StringBuilder(sbCapacity); for (Object partitionId : partIdList) { partSb.append(extractSqlLong(partitionId)).append(","); }/* ww w. j a v a 2s .c o m*/ String partIds = trimCommaList(partSb); // Get most of the fields for the IDs provided. // Assume db and table names are the same for all partition, as provided in arguments. String queryText = "select " + PARTITIONS + ".\"PART_ID\", " + SDS + ".\"SD_ID\", " + SDS + ".\"CD_ID\"," + " " + SERDES + ".\"SERDE_ID\", " + PARTITIONS + ".\"CREATE_TIME\"," + " " + PARTITIONS + ".\"LAST_ACCESS_TIME\", " + SDS + ".\"INPUT_FORMAT\", " + SDS + ".\"IS_COMPRESSED\"," + " " + SDS + ".\"IS_STOREDASSUBDIRECTORIES\", " + SDS + ".\"LOCATION\", " + SDS + ".\"NUM_BUCKETS\"," + " " + SDS + ".\"OUTPUT_FORMAT\", " + SERDES + ".\"NAME\", " + SERDES + ".\"SLIB\" " + "from " + PARTITIONS + "" + " left outer join " + SDS + " on " + PARTITIONS + ".\"SD_ID\" = " + SDS + ".\"SD_ID\" " + " left outer join " + SERDES + " on " + SDS + ".\"SERDE_ID\" = " + SERDES + ".\"SERDE_ID\" " + "where \"PART_ID\" in (" + partIds + ") order by \"PART_NAME\" asc"; long start = doTrace ? System.nanoTime() : 0; Query query = pm.newQuery("javax.jdo.query.SQL", queryText); List<Object[]> sqlResult = executeWithArray(query, null, queryText); long queryTime = doTrace ? System.nanoTime() : 0; Deadline.checkTimeout(); // Read all the fields and create partitions, SDs and serdes. TreeMap<Long, Partition> partitions = new TreeMap<Long, Partition>(); TreeMap<Long, StorageDescriptor> sds = new TreeMap<Long, StorageDescriptor>(); TreeMap<Long, SerDeInfo> serdes = new TreeMap<Long, SerDeInfo>(); TreeMap<Long, List<FieldSchema>> colss = new TreeMap<Long, List<FieldSchema>>(); // Keep order by name, consistent with JDO. ArrayList<Partition> orderedResult = new ArrayList<Partition>(partIdList.size()); // Prepare StringBuilder-s for "in (...)" lists to use in one-to-many queries. StringBuilder sdSb = new StringBuilder(sbCapacity), serdeSb = new StringBuilder(sbCapacity); StringBuilder colsSb = new StringBuilder(7); // We expect that there's only one field schema. tblName = tblName.toLowerCase(); dbName = dbName.toLowerCase(); for (Object[] fields : sqlResult) { // Here comes the ugly part... long partitionId = extractSqlLong(fields[0]); Long sdId = extractSqlLong(fields[1]); Long colId = extractSqlLong(fields[2]); Long serdeId = extractSqlLong(fields[3]); // A partition must have at least sdId and serdeId set, or nothing set if it's a view. if (sdId == null || serdeId == null) { if (isView == null) { isView = isViewTable(dbName, tblName); } if ((sdId != null || colId != null || serdeId != null) || !isView) { throw new MetaException("Unexpected null for one of the IDs, SD " + sdId + ", serde " + serdeId + " for a " + (isView ? "" : "non-") + " view"); } } Partition part = new Partition(); orderedResult.add(part); // Set the collection fields; some code might not check presence before accessing them. part.setParameters(new HashMap<String, String>()); part.setValues(new ArrayList<String>()); part.setDbName(dbName); part.setTableName(tblName); if (fields[4] != null) part.setCreateTime(extractSqlInt(fields[4])); if (fields[5] != null) part.setLastAccessTime(extractSqlInt(fields[5])); partitions.put(partitionId, part); if (sdId == null) continue; // Probably a view. assert serdeId != null; // We assume each partition has an unique SD. StorageDescriptor sd = new StorageDescriptor(); StorageDescriptor oldSd = sds.put(sdId, sd); if (oldSd != null) { throw new MetaException("Partitions reuse SDs; we don't expect that"); } // Set the collection fields; some code might not check presence before accessing them. sd.setSortCols(new ArrayList<Order>()); sd.setBucketCols(new ArrayList<String>()); sd.setParameters(new HashMap<String, String>()); sd.setSkewedInfo(new SkewedInfo(new ArrayList<String>(), new ArrayList<List<String>>(), new HashMap<List<String>, String>())); sd.setInputFormat((String) fields[6]); Boolean tmpBoolean = extractSqlBoolean(fields[7]); if (tmpBoolean != null) sd.setCompressed(tmpBoolean); tmpBoolean = extractSqlBoolean(fields[8]); if (tmpBoolean != null) sd.setStoredAsSubDirectories(tmpBoolean); sd.setLocation((String) fields[9]); if (fields[10] != null) sd.setNumBuckets(extractSqlInt(fields[10])); sd.setOutputFormat((String) fields[11]); sdSb.append(sdId).append(","); part.setSd(sd); if (colId != null) { List<FieldSchema> cols = colss.get(colId); // We expect that colId will be the same for all (or many) SDs. if (cols == null) { cols = new ArrayList<FieldSchema>(); colss.put(colId, cols); colsSb.append(colId).append(","); } sd.setCols(cols); } // We assume each SD has an unique serde. SerDeInfo serde = new SerDeInfo(); SerDeInfo oldSerde = serdes.put(serdeId, serde); if (oldSerde != null) { throw new MetaException("SDs reuse serdes; we don't expect that"); } serde.setParameters(new HashMap<String, String>()); serde.setName((String) fields[12]); serde.setSerializationLib((String) fields[13]); serdeSb.append(serdeId).append(","); sd.setSerdeInfo(serde); Deadline.checkTimeout(); } query.closeAll(); timingTrace(doTrace, queryText, start, queryTime); // Now get all the one-to-many things. Start with partitions. queryText = "select \"PART_ID\", \"PARAM_KEY\", \"PARAM_VALUE\" from " + PARTITION_PARAMS + "" + " where \"PART_ID\" in (" + partIds + ") and \"PARAM_KEY\" is not null" + " order by \"PART_ID\" asc"; loopJoinOrderedResult(partitions, queryText, 0, new ApplyFunc<Partition>() { @Override public void apply(Partition t, Object[] fields) { t.putToParameters((String) fields[1], (String) fields[2]); } }); // Perform conversion of null map values for (Partition t : partitions.values()) { t.setParameters(MetaStoreUtils.trimMapNulls(t.getParameters(), convertMapNullsToEmptyStrings)); } queryText = "select \"PART_ID\", \"PART_KEY_VAL\" from " + PARTITION_KEY_VALS + "" + " where \"PART_ID\" in (" + partIds + ") and \"INTEGER_IDX\" >= 0" + " order by \"PART_ID\" asc, \"INTEGER_IDX\" asc"; loopJoinOrderedResult(partitions, queryText, 0, new ApplyFunc<Partition>() { @Override public void apply(Partition t, Object[] fields) { t.addToValues((String) fields[1]); } }); // Prepare IN (blah) lists for the following queries. Cut off the final ','s. if (sdSb.length() == 0) { assert serdeSb.length() == 0 && colsSb.length() == 0; return orderedResult; // No SDs, probably a view. } String sdIds = trimCommaList(sdSb); String serdeIds = trimCommaList(serdeSb); String colIds = trimCommaList(colsSb); // Get all the stuff for SD. Don't do empty-list check - we expect partitions do have SDs. queryText = "select \"SD_ID\", \"PARAM_KEY\", \"PARAM_VALUE\" from " + SD_PARAMS + "" + " where \"SD_ID\" in (" + sdIds + ") and \"PARAM_KEY\" is not null" + " order by \"SD_ID\" asc"; loopJoinOrderedResult(sds, queryText, 0, new ApplyFunc<StorageDescriptor>() { @Override public void apply(StorageDescriptor t, Object[] fields) { t.putToParameters((String) fields[1], extractSqlClob(fields[2])); } }); // Perform conversion of null map values for (StorageDescriptor t : sds.values()) { t.setParameters(MetaStoreUtils.trimMapNulls(t.getParameters(), convertMapNullsToEmptyStrings)); } queryText = "select \"SD_ID\", \"COLUMN_NAME\", " + SORT_COLS + ".\"ORDER\"" + " from " + SORT_COLS + "" + " where \"SD_ID\" in (" + sdIds + ") and \"INTEGER_IDX\" >= 0" + " order by \"SD_ID\" asc, \"INTEGER_IDX\" asc"; loopJoinOrderedResult(sds, queryText, 0, new ApplyFunc<StorageDescriptor>() { @Override public void apply(StorageDescriptor t, Object[] fields) { if (fields[2] == null) return; t.addToSortCols(new Order((String) fields[1], extractSqlInt(fields[2]))); } }); queryText = "select \"SD_ID\", \"BUCKET_COL_NAME\" from " + BUCKETING_COLS + "" + " where \"SD_ID\" in (" + sdIds + ") and \"INTEGER_IDX\" >= 0" + " order by \"SD_ID\" asc, \"INTEGER_IDX\" asc"; loopJoinOrderedResult(sds, queryText, 0, new ApplyFunc<StorageDescriptor>() { @Override public void apply(StorageDescriptor t, Object[] fields) { t.addToBucketCols((String) fields[1]); } }); // Skewed columns stuff. queryText = "select \"SD_ID\", \"SKEWED_COL_NAME\" from " + SKEWED_COL_NAMES + "" + " where \"SD_ID\" in (" + sdIds + ") and \"INTEGER_IDX\" >= 0" + " order by \"SD_ID\" asc, \"INTEGER_IDX\" asc"; boolean hasSkewedColumns = loopJoinOrderedResult(sds, queryText, 0, new ApplyFunc<StorageDescriptor>() { @Override public void apply(StorageDescriptor t, Object[] fields) { if (!t.isSetSkewedInfo()) t.setSkewedInfo(new SkewedInfo()); t.getSkewedInfo().addToSkewedColNames((String) fields[1]); } }) > 0; // Assume we don't need to fetch the rest of the skewed column data if we have no columns. if (hasSkewedColumns) { // We are skipping the SKEWED_STRING_LIST table here, as it seems to be totally useless. queryText = "select " + SKEWED_VALUES + ".\"SD_ID_OID\"," + " " + SKEWED_STRING_LIST_VALUES + ".\"STRING_LIST_ID\"," + " " + SKEWED_STRING_LIST_VALUES + ".\"STRING_LIST_VALUE\" " + "from " + SKEWED_VALUES + " " + " left outer join " + SKEWED_STRING_LIST_VALUES + " on " + SKEWED_VALUES + "." + "\"STRING_LIST_ID_EID\" = " + SKEWED_STRING_LIST_VALUES + ".\"STRING_LIST_ID\" " + "where " + SKEWED_VALUES + ".\"SD_ID_OID\" in (" + sdIds + ") " + " and " + SKEWED_VALUES + ".\"STRING_LIST_ID_EID\" is not null " + " and " + SKEWED_VALUES + ".\"INTEGER_IDX\" >= 0 " + "order by " + SKEWED_VALUES + ".\"SD_ID_OID\" asc, " + SKEWED_VALUES + ".\"INTEGER_IDX\" asc," + " " + SKEWED_STRING_LIST_VALUES + ".\"INTEGER_IDX\" asc"; loopJoinOrderedResult(sds, queryText, 0, new ApplyFunc<StorageDescriptor>() { private Long currentListId; private List<String> currentList; @Override public void apply(StorageDescriptor t, Object[] fields) throws MetaException { if (!t.isSetSkewedInfo()) t.setSkewedInfo(new SkewedInfo()); // Note that this is not a typical list accumulator - there's no call to finalize // the last list. Instead we add list to SD first, as well as locally to add elements. if (fields[1] == null) { currentList = null; // left outer join produced a list with no values currentListId = null; t.getSkewedInfo().addToSkewedColValues(Collections.<String>emptyList()); } else { long fieldsListId = extractSqlLong(fields[1]); if (currentListId == null || fieldsListId != currentListId) { currentList = new ArrayList<String>(); currentListId = fieldsListId; t.getSkewedInfo().addToSkewedColValues(currentList); } currentList.add((String) fields[2]); } } }); // We are skipping the SKEWED_STRING_LIST table here, as it seems to be totally useless. queryText = "select " + SKEWED_COL_VALUE_LOC_MAP + ".\"SD_ID\"," + " " + SKEWED_STRING_LIST_VALUES + ".STRING_LIST_ID," + " " + SKEWED_COL_VALUE_LOC_MAP + ".\"LOCATION\"," + " " + SKEWED_STRING_LIST_VALUES + ".\"STRING_LIST_VALUE\" " + "from " + SKEWED_COL_VALUE_LOC_MAP + "" + " left outer join " + SKEWED_STRING_LIST_VALUES + " on " + SKEWED_COL_VALUE_LOC_MAP + "." + "\"STRING_LIST_ID_KID\" = " + SKEWED_STRING_LIST_VALUES + ".\"STRING_LIST_ID\" " + "where " + SKEWED_COL_VALUE_LOC_MAP + ".\"SD_ID\" in (" + sdIds + ")" + " and " + SKEWED_COL_VALUE_LOC_MAP + ".\"STRING_LIST_ID_KID\" is not null " + "order by " + SKEWED_COL_VALUE_LOC_MAP + ".\"SD_ID\" asc," + " " + SKEWED_STRING_LIST_VALUES + ".\"STRING_LIST_ID\" asc," + " " + SKEWED_STRING_LIST_VALUES + ".\"INTEGER_IDX\" asc"; loopJoinOrderedResult(sds, queryText, 0, new ApplyFunc<StorageDescriptor>() { private Long currentListId; private List<String> currentList; @Override public void apply(StorageDescriptor t, Object[] fields) throws MetaException { if (!t.isSetSkewedInfo()) { SkewedInfo skewedInfo = new SkewedInfo(); skewedInfo.setSkewedColValueLocationMaps(new HashMap<List<String>, String>()); t.setSkewedInfo(skewedInfo); } Map<List<String>, String> skewMap = t.getSkewedInfo().getSkewedColValueLocationMaps(); // Note that this is not a typical list accumulator - there's no call to finalize // the last list. Instead we add list to SD first, as well as locally to add elements. if (fields[1] == null) { currentList = new ArrayList<String>(); // left outer join produced a list with no values currentListId = null; } else { long fieldsListId = extractSqlLong(fields[1]); if (currentListId == null || fieldsListId != currentListId) { currentList = new ArrayList<String>(); currentListId = fieldsListId; } else { skewMap.remove(currentList); // value based compare.. remove first } currentList.add((String) fields[3]); } skewMap.put(currentList, (String) fields[2]); } }); } // if (hasSkewedColumns) // Get FieldSchema stuff if any. if (!colss.isEmpty()) { // We are skipping the CDS table here, as it seems to be totally useless. queryText = "select \"CD_ID\", \"COMMENT\", \"COLUMN_NAME\", \"TYPE_NAME\"" + " from " + COLUMNS_V2 + " where \"CD_ID\" in (" + colIds + ") and \"INTEGER_IDX\" >= 0" + " order by \"CD_ID\" asc, \"INTEGER_IDX\" asc"; loopJoinOrderedResult(colss, queryText, 0, new ApplyFunc<List<FieldSchema>>() { @Override public void apply(List<FieldSchema> t, Object[] fields) { t.add(new FieldSchema((String) fields[2], extractSqlClob(fields[3]), (String) fields[1])); } }); } // Finally, get all the stuff for serdes - just the params. queryText = "select \"SERDE_ID\", \"PARAM_KEY\", \"PARAM_VALUE\" from " + SERDE_PARAMS + "" + " where \"SERDE_ID\" in (" + serdeIds + ") and \"PARAM_KEY\" is not null" + " order by \"SERDE_ID\" asc"; loopJoinOrderedResult(serdes, queryText, 0, new ApplyFunc<SerDeInfo>() { @Override public void apply(SerDeInfo t, Object[] fields) { t.putToParameters((String) fields[1], extractSqlClob(fields[2])); } }); // Perform conversion of null map values for (SerDeInfo t : serdes.values()) { t.setParameters(MetaStoreUtils.trimMapNulls(t.getParameters(), convertMapNullsToEmptyStrings)); } return orderedResult; }
From source file:br.org.indt.ndg.common.ResultParser.java
private ResultXml processResult() { TreeMap<Integer, Category> categories; ResultXml result = new ResultXml(); result.setXmldoc(document);/* ww w . j a v a 2 s . c o m*/ logger.info("loading result attributes"); NodeList nodeSurvey = document.getElementsByTagName("result"); NamedNodeMap resultAttr = nodeSurvey.item(0).getAttributes(); result.setResultId(resultAttr.getNamedItem("r_id").getNodeValue()); result.setSurveyId(resultAttr.getNamedItem("s_id").getNodeValue()); result.setImei(resultAttr.getNamedItem("u_id").getNodeValue()); result.setTime(resultAttr.getNamedItem("time").getNodeValue()); NodeList resultChild = nodeSurvey.item(0).getChildNodes(); for (int i = 0; i < resultChild.getLength(); i++) { if (resultChild.item(i).getNodeName().equals("title")) { result.setTitle(resultChild.item(i).getTextContent()); } else if (resultChild.item(i).getNodeName().equals("latitude")) { result.setLatitude(resultChild.item(i).getTextContent()); } else if (resultChild.item(i).getNodeName().equals("longitude")) { result.setLongitude(resultChild.item(i).getTextContent()); } } logger.info("loading categories attributes"); categories = new TreeMap<Integer, Category>(); NodeList nodesCategory = document.getElementsByTagName("category"); int countCategory = nodesCategory.getLength(); for (int c = 0; c < countCategory; ++c) { Category category = new Category(); NamedNodeMap categoryAttr = nodesCategory.item(c).getAttributes(); category.setId(Integer.parseInt(categoryAttr.getNamedItem("id").getNodeValue())); category.setName(categoryAttr.getNamedItem("name").getNodeValue()); categories.put(category.getId(), category); } logger.info("loading answers attributes"); NodeList nodesAnswer = document.getElementsByTagName("answer"); int countAnswer = nodesAnswer.getLength(); logger.info("countAnswer = " + nodesAnswer); for (int a = 0; a < countAnswer; ++a) { NamedNodeMap answerAttr = nodesAnswer.item(a).getAttributes(); Field answer = new Field(); String answerId = answerAttr.getNamedItem("id").getNodeValue(); answer.setId(Integer.parseInt(answerId)); String answerType = answerAttr.getNamedItem("type").getNodeValue(); answer.setXmlType(answerType); if (answer.getXmlType().equals("_choice")) { NodeList answerChild = nodesAnswer.item(a).getChildNodes(); Choice choice = new Choice(); for (int i = 0; i < answerChild.getLength(); i++) { if (answerChild.item(i).getNodeName().equals("item")) { try { int index = Integer.parseInt(answerChild.item(i).getTextContent()); Item item = new Item(); item.setIndex(index); item.setOtr("0"); choice.addItem(item); } catch (Exception e) { logger.error("Choice with <item></item>, must have a value " + e); } } else if (answerChild.item(i).getNodeName().equals("other")) { try { NamedNodeMap itemAttr = answerChild.item(i).getAttributes(); int index = Integer.parseInt(itemAttr.getNamedItem("index").getNodeValue()); String value = answerChild.item(i).getTextContent(); Item item = new Item(); item.setIndex(index); item.setValue(value); item.setOtr("1"); choice.addItem(item); } catch (Exception e) { logger.error("Other with <index></index>, must have a value " + e); } } } answer.setChoice(choice); } else { String value = null; try { Node nodeAnswerItem = nodesAnswer.item(a); if (nodeAnswerItem != null) { NodeList nodeList = nodeAnswerItem.getChildNodes(); Node nodeAnswerItemItem = nodeList.item(1); value = nodeAnswerItemItem.getTextContent(); // value = nodesAnswer.item(a).getChildNodes().item(1).getTextContent(); } else { value = ""; } } catch (Exception e) { value = ""; } answer.setValue(value); logger.info("Value = " + value); } String categoryName = nodesAnswer.item(a).getParentNode().getAttributes().getNamedItem("name") .getNodeValue(); int categoryId = Integer.parseInt( nodesAnswer.item(a).getParentNode().getAttributes().getNamedItem("id").getNodeValue()); logger.debug(categoryName + " | " + answerId + " | " + answerType); Category category = categories.get(categoryId); logger.debug("category name: " + category.getName()); answer.setCategoryId(categoryId); category.addField(answer); categories.put(categoryId, category); } result.setCategories(categories); return result; }