List of usage examples for org.hibernate CacheMode IGNORE
CacheMode IGNORE
To view the source code for org.hibernate CacheMode IGNORE.
Click Source Link
From source file:ubic.gemma.persistence.service.association.coexpression.CoexpressionDaoImpl.java
License:Apache License
@Override public void createOrUpdate(BioAssaySet bioAssaySet, List<NonPersistentNonOrderedCoexpLink> links, LinkCreator c, Set<Gene> genesTested) { // assumption is that these are _all_ the links for this experiment assert !links.isEmpty(); assert bioAssaySet != null; assert c != null; Collections.sort(links);//w ww. j a v a2 s .c o m Session sess = this.getSessionFactory().getCurrentSession(); sess.setCacheMode(CacheMode.IGNORE); // to determine the species Gene gene = (Gene) sess.get(Gene.class, links.iterator().next().getFirstGene()); String geneLinkClassName = CoexpressionQueryUtils.getGeneLinkClassName(gene); /* * Check that there are no links for this experiment. */ if (this.countLinks(gene.getTaxon(), bioAssaySet) > 0) { throw new IllegalStateException( "There are already links for given bioAssaySet; they must be deleted before proceeding"); } /* * Attempt to save database trips */ Map<NonPersistentNonOrderedCoexpLink, Boolean> existingResults = this.preFetch(links); String s = "from " + geneLinkClassName + " where firstGene =:f and secondGene=:s and positiveCorrelation=:pc"; Query q = sess.createQuery(s); SQLQuery updateFlippedLinkQuery = sess .createSQLQuery("UPDATE " + CoexpressionQueryUtils.getGeneLinkTableName(gene.getTaxon()) + " SET SUPPORT=:s WHERE FIRST_GENE_FK=:g2 AND SECOND_GENE_FK=:g1 AND POSITIVE=:po"); // map of linkid to links, for establishing the EE-level links. TreeMap<Long, NonPersistentNonOrderedCoexpLink> linkIds = new TreeMap<>(); // keep order so for this experiment // they are in order. Set<Long> seenExistingLinks = new HashSet<>(); // for sanity checks. Set<NonPersistentNonOrderedCoexpLink> seenNewLinks = new HashSet<>(); // for sanity checks. Set<SupportDetails> seenNewSupportDetails = new HashSet<>(); // for sanity checks. int numNew = 0; int numUpdated = 0; int progress = 0; int BATCH_SIZE = 1024; // make a multiple of jdbc batch size... Map<SupportDetails, Gene2GeneCoexpression> batchToCreate = new LinkedHashMap<>(); List<Gene2GeneCoexpression> newFlippedLinks = new ArrayList<>(); Set<Long> genesWithUpdatedData = new HashSet<>(); sess.flush(); sess.clear(); // for each link see if there is already an entry; make a new one if necessary or update the old one. CoexpressionDaoImpl.log.info("Starting link processing"); for (NonPersistentNonOrderedCoexpLink proposedG2G : links) { Long firstGene = proposedG2G.getFirstGene(); Long secondGene = proposedG2G.getSecondGene(); // There is an index for f+s, but querying one-at-a-time is going to be slow. I attempted to speed it up by // fetching all links for a gene when we see it, but this causes problems with data being stale. Prefetching // with just the ability to tell if a link is new or not takes a lot of memory and doesn't speed things up // much. Trying keeping an index of which links a gene has, so we know whether we need to check the database // or not. // // Currently it takes about 1 minute to process 10k links on a relatively small database, much of this is // the findLink call. Gene2GeneCoexpression existingLink = this.findLink(q, proposedG2G, existingResults); /* * To speed this up? * * - Fetch all links for a gene in one batch, instead of looping over them one at a time. The problem is the * flipped links involve other genes that we fetch later in the same transaction, and this all has to be * done in one transaction. I experimented with this already */ if (existingLink == null) { // initialize the supportdetails SupportDetails sd = c.createSupportDetails(firstGene, secondGene, proposedG2G.isPositiveCorrelation()); sd.addEntity(bioAssaySet.getId()); assert sd.getNumIds() > 0; assert sd.isIncluded(bioAssaySet.getId()); // Must be unique assert !seenNewSupportDetails.contains(sd) : "Already saw " + sd + " while processing " + proposedG2G; assert proposedG2G.getLink() != null; batchToCreate.put(sd, proposedG2G.getLink()); if (seenNewLinks.contains(proposedG2G)) { CoexpressionDaoImpl.log.warn( "The data passed had the same new link represented more than once: " + proposedG2G); continue; } seenNewSupportDetails.add(sd); seenNewLinks.add(proposedG2G); if (CoexpressionDaoImpl.log.isDebugEnabled()) CoexpressionDaoImpl.log.debug("New: " + proposedG2G); numNew++; } else { // This code assumes that the flipped version is in the database, but we don't retrieve it // yet. also note that the support of the existing link could be zero, if DELETE_ORPHAN_LINKS = false // (or if initializeLinksFromExistingData was used) // Sanity check. If this happens, there must be two versions of the same link already in the input. if (seenExistingLinks.contains(existingLink.getId())) { throw new IllegalStateException( "The data passed had the same existing link represented more than once: " + existingLink); } /* sanity check that we aren't adding dataset twice; we might be able make this an assertion instead. */ if (existingLink.isSupportedBy(bioAssaySet)) { throw new IllegalStateException("Support for this experiment already exists for " + existingLink + ", must be deleted first"); } // cache old support for sanity check int oldSupport = existingLink.getSupportDetails().getNumIds(); // update the support existingLink.getSupportDetails().addEntity(bioAssaySet.getId()); existingLink.updateNumDatasetsSupporting(); // there is no cascade... on purpose. sess.update(existingLink.getSupportDetails()); assert oldSupport + 1 == existingLink.getNumDatasetsSupporting(); assert existingLink.getSupportDetails().getNumIds() == oldSupport + 1; // track so we add corresponding Experiment-level links later. linkIds.put(existingLink.getId(), new NonPersistentNonOrderedCoexpLink(existingLink)); seenExistingLinks.add(existingLink.getId()); /* * The flipped link is asserted to be in the database. The support details is already dealt with; we * just have to update the support value. */ int numFlippedUpdated = updateFlippedLinkQuery .setParameter("s", existingLink.getNumDatasetsSupporting()) .setParameter("g2", proposedG2G.getSecondGene()) .setParameter("g1", proposedG2G.getFirstGene()) .setParameter("po", proposedG2G.isPositiveCorrelation() ? 1 : 0).executeUpdate(); assert numFlippedUpdated == 1 : "Flipped link missing for " + proposedG2G + " [" + numFlippedUpdated + "]"; numUpdated++; if (CoexpressionDaoImpl.log.isDebugEnabled()) CoexpressionDaoImpl.log.debug("Updated: " + proposedG2G); } genesWithUpdatedData.add(firstGene); genesWithUpdatedData.add(secondGene); if (++progress % 5000 == 0) { CoexpressionDaoImpl.log.info("Processed " + progress + "/" + links.size() + " gene-level links..." + numUpdated + " updated, " + numNew + " new"); } if (batchToCreate.size() >= BATCH_SIZE) { newFlippedLinks.addAll(this.saveBatchAndMakeFlipped(sess, linkIds, batchToCreate, c)); } else if (numUpdated > 0 && numUpdated % BATCH_SIZE == 0) { sess.flush(); sess.clear(); } } // loop over links // tail end batch if (!batchToCreate.isEmpty()) { // we make the flipped links later to optimize their ordering. newFlippedLinks.addAll(this.saveBatchAndMakeFlipped(sess, linkIds, batchToCreate, c)); } // flush the updated ones one last time... if (numUpdated > 0) { sess.flush(); sess.clear(); } assert links.size() == linkIds.size(); CoexpressionDaoImpl.log.info(numUpdated + " updated, " + numNew + " new links"); /* * sort and save the accumulated new flipped versions of the new links, which reuse the supportDetails. In the * flipped links, the first gene is the second gene and vice versa. Continue to accumulate the flipped links. */ CoexpressionDaoImpl.log.info("Saving " + newFlippedLinks.size() + " flipped versions of new links ..."); Collections.sort(newFlippedLinks, new Comparator<Gene2GeneCoexpression>() { @Override public int compare(Gene2GeneCoexpression o1, Gene2GeneCoexpression o2) { return o1.getFirstGene().compareTo(o2.getFirstGene()); } }); progress = 0; for (Gene2GeneCoexpression gl : newFlippedLinks) { sess.save(gl); if (++progress % 5000 == 0) { CoexpressionDaoImpl.log.info("Processed " + progress + "/" + newFlippedLinks.size() + " new flipped gene-level links..."); } if (progress % BATCH_SIZE == 0) { sess.flush(); sess.clear(); } } /* * Save experiment-level links */ CoexpressionDaoImpl.log .info("Saving " + linkIds.size() + " experiment-level links (plus flipped versions) ..."); this.saveExperimentLevelLinks(sess, c, linkIds, bioAssaySet); if (genesTested != null) this.updatedTestedIn(bioAssaySet, genesTested); this.updateGeneCoexpressedWith(links); // kick anything we updated out of the cache. int numRemovedFromCache = this.gene2GeneCoexpressionCache.remove(genesWithUpdatedData); if (numRemovedFromCache > 0) CoexpressionDaoImpl.log.info(numRemovedFromCache + " results evicted from cache"); // flush happens on commit... CoexpressionDaoImpl.log.info("Done, flushing changes ..."); }
From source file:ubic.gemma.persistence.service.association.coexpression.CoexpressionDaoImpl.java
License:Apache License
@Override @Transactional/*from www . j a v a2s . c o m*/ public void deleteLinks(Taxon t, BioAssaySet experiment) { Session sess = this.getSessionFactory().getCurrentSession(); sess.setCacheMode(CacheMode.IGNORE); CoexpressionDaoImpl.log.info("Fetching any old coexpression ..."); Collection<Gene2GeneCoexpression> links = this.getCoexpression(t, experiment); Set<NonPersistentNonOrderedCoexpLink> toRemove = new HashSet<>(); // even if there are no links, we shouldn't assume we can bail; the 'tested-in' information might be there. if (!links.isEmpty()) { CoexpressionDaoImpl.log.info("Removing coexpression information for " + experiment + "; updating " + links.size() + " links (count includes flipped versions)."); // adjust gene-level links int count = 0; int numWithZeroSupportLeft = 0; int BATCH_SIZE = 1024; Collection<SupportDetails> supportDetailsToDelete = new HashSet<>(); Collection<SupportDetails> supportDetailsToUpdate = new HashSet<>(); Collection<Long> genesAffected = new HashSet<>(); for (Gene2GeneCoexpression g2g : links) { genesAffected.add(g2g.getFirstGene()); genesAffected.add(g2g.getSecondGene()); // decrement support; details are shared by both links, just update it once! SupportDetails sd = g2g.getSupportDetails(); if (!supportDetailsToUpdate.contains(sd) && !supportDetailsToDelete.contains(sd)) { /* * If we already saw the supportDetails it might already be zero. But if we didn't, it can't. */ assert g2g.getNumDatasetsSupporting() > 0 : "Support was " + g2g.getNumDatasetsSupporting() + " for " + g2g; sd.removeEntity(experiment.getId()); assert !sd.getIds().contains(experiment.getId()); supportDetailsToUpdate.add(sd); } g2g.updateNumDatasetsSupporting(); assert g2g.getNumDatasetsSupporting() >= 0; if (g2g.getNumDatasetsSupporting() == 0) { /* * we might still want to keep it, on the presumption that it will get filled back in. */ if (CoexpressionDaoImpl.DELETE_ORPHAN_LINKS) { sess.delete(g2g); // it might be in here already (flipped), but that's okay. supportDetailsToDelete.add(sd); // from the quickindex. But leave it there otherwise. toRemove.add(new NonPersistentNonOrderedCoexpLink(g2g)); } else { sess.update(g2g); } numWithZeroSupportLeft++; } else { sess.update(g2g); } if (++count % 10000 == 0) { CoexpressionDaoImpl.log .info("Removed support for " + count + " links for " + experiment + "..."); } if (count % BATCH_SIZE == 0) { sess.flush(); sess.clear(); } } sess.flush(); sess.clear(); this.updateModifiedSupportDetails(experiment, supportDetailsToDelete, supportDetailsToUpdate); if (CoexpressionDaoImpl.DELETE_ORPHAN_LINKS) { CoexpressionDaoImpl.log.info("Adjusted " + links.size() + " gene-level links supported by the experiment; " + numWithZeroSupportLeft + " links removed from the system as support dropped to zero."); } else { CoexpressionDaoImpl.log.info("Adjusted " + links.size() + " gene-level links supported by the experiment; " + numWithZeroSupportLeft + " gene-level links now have support dropped to zero but they were left in place"); } // remove the ExperimentCoexpressionLinks int numDeleted = sess.createQuery( "delete from " + CoexpressionQueryUtils.getExperimentLinkClassName(t) + " where experiment=:ee") .setParameter("ee", experiment).executeUpdate(); CoexpressionDaoImpl.log.info("Deleted " + numDeleted + " experiment-level links"); // invalidate the cache. int numRemovedFromCache = gene2GeneCoexpressionCache.remove(genesAffected); if (numRemovedFromCache > 0) CoexpressionDaoImpl.log.info(numRemovedFromCache + " results evicted from cache"); } // we do NOT redo the node degree information, which will be refreshed "periodically" // we always have to do this, even if there are no links. this.removeTestedIn(t, experiment); // update our quick index if (!toRemove.isEmpty()) this.removeCoexpressedWith(toRemove); }