List of usage examples for java.util TreeMap entrySet
EntrySet entrySet
To view the source code for java.util TreeMap entrySet.
Click Source Link
From source file:org.apache.hadoop.hbase.master.LoadBalancer.java
/** * Generate a global load balancing plan according to the specified map of * server information to the most loaded regions of each server. * * The load balancing invariant is that all servers are within 1 region of the * average number of regions per server. If the average is an integer number, * all servers will be balanced to the average. Otherwise, all servers will * have either floor(average) or ceiling(average) regions. * * The algorithm is currently implemented as such: * * <ol>/*from ww w .j ava 2s .c o m*/ * <li>Determine the two valid numbers of regions each server should have, * <b>MIN</b>=floor(average) and <b>MAX</b>=ceiling(average). * * <li>Iterate down the most loaded servers, shedding regions from each so * each server hosts exactly <b>MAX</b> regions. Stop once you reach a * server that already has <= <b>MAX</b> regions. * <p> * Order the regions to move from most recent to least. * * <li>Iterate down the least loaded servers, assigning regions so each server * has exactly </b>MIN</b> regions. Stop once you reach a server that * already has >= <b>MIN</b> regions. * * Regions being assigned to underloaded servers are those that were shed * in the previous step. It is possible that there were not enough * regions shed to fill each underloaded server to <b>MIN</b>. If so we * end up with a number of regions required to do so, <b>neededRegions</b>. * * It is also possible that we were able fill each underloaded but ended * up with regions that were unassigned from overloaded servers but that * still do not have assignment. * * If neither of these conditions hold (no regions needed to fill the * underloaded servers, no regions leftover from overloaded servers), * we are done and return. Otherwise we handle these cases below. * * <li>If <b>neededRegions</b> is non-zero (still have underloaded servers), * we iterate the most loaded servers again, shedding a single server from * each (this brings them from having <b>MAX</b> regions to having * <b>MIN</b> regions). * * <li>We now definitely have more regions that need assignment, either from * the previous step or from the original shedding from overloaded servers. * * Iterate the least loaded servers filling each to <b>MIN</b>. * * <li>If we still have more regions that need assignment, again iterate the * least loaded servers, this time giving each one (filling them to * </b>MAX</b>) until we run out. * * <li>All servers will now either host <b>MIN</b> or <b>MAX</b> regions. * * In addition, any server hosting >= <b>MAX</b> regions is guaranteed * to end up with <b>MAX</b> regions at the end of the balancing. This * ensures the minimal number of regions possible are moved. * </ol> * * TODO: We can at-most reassign the number of regions away from a particular * server to be how many they report as most loaded. * Should we just keep all assignment in memory? Any objections? * Does this mean we need HeapSize on HMaster? Or just careful monitor? * (current thinking is we will hold all assignments in memory) * * @param clusterState Map of regionservers and their load/region information to * a list of their most loaded regions * @return a list of regions to be moved, including source and destination, * or null if cluster is already balanced */ public List<RegionPlan> balanceCluster(Map<HServerInfo, List<HRegionInfo>> clusterState) { long startTime = System.currentTimeMillis(); // Make a map sorted by load and count regions TreeMap<HServerInfo, List<HRegionInfo>> serversByLoad = new TreeMap<HServerInfo, List<HRegionInfo>>( new HServerInfo.LoadComparator()); int numServers = clusterState.size(); if (numServers == 0) { LOG.debug("numServers=0 so skipping load balancing"); return null; } int numRegions = 0; // Iterate so we can count regions as we build the map for (Map.Entry<HServerInfo, List<HRegionInfo>> server : clusterState.entrySet()) { server.getKey().getLoad().setNumberOfRegions(server.getValue().size()); numRegions += server.getKey().getLoad().getNumberOfRegions(); serversByLoad.put(server.getKey(), server.getValue()); } // Check if we even need to do any load balancing float average = (float) numRegions / numServers; // for logging // HBASE-3681 check sloppiness first int floor = (int) Math.floor(average * (1 - slop)); int ceiling = (int) Math.ceil(average * (1 + slop)); if (serversByLoad.lastKey().getLoad().getNumberOfRegions() <= ceiling && serversByLoad.firstKey().getLoad().getNumberOfRegions() >= floor) { // Skipped because no server outside (min,max) range LOG.info("Skipping load balancing. servers=" + numServers + " " + "regions=" + numRegions + " average=" + average + " " + "mostloaded=" + serversByLoad.lastKey().getLoad().getNumberOfRegions() + " leastloaded=" + serversByLoad.lastKey().getLoad().getNumberOfRegions()); return null; } int min = numRegions / numServers; int max = numRegions % numServers == 0 ? min : min + 1; // Balance the cluster // TODO: Look at data block locality or a more complex load to do this List<RegionPlan> regionsToMove = new ArrayList<RegionPlan>(); int regionidx = 0; // track the index in above list for setting destination // Walk down most loaded, pruning each to the max int serversOverloaded = 0; Map<HServerInfo, BalanceInfo> serverBalanceInfo = new TreeMap<HServerInfo, BalanceInfo>(); for (Map.Entry<HServerInfo, List<HRegionInfo>> server : serversByLoad.descendingMap().entrySet()) { HServerInfo serverInfo = server.getKey(); int regionCount = serverInfo.getLoad().getNumberOfRegions(); if (regionCount <= max) { serverBalanceInfo.put(serverInfo, new BalanceInfo(0, 0)); break; } serversOverloaded++; List<HRegionInfo> regions = randomize(server.getValue()); int numToOffload = Math.min(regionCount - max, regions.size()); int numTaken = 0; for (int i = regions.size() - 1; i >= 0; i--) { HRegionInfo hri = regions.get(i); // Don't rebalance meta regions. if (hri.isMetaRegion()) continue; regionsToMove.add(new RegionPlan(hri, serverInfo, null)); numTaken++; if (numTaken >= numToOffload) break; } serverBalanceInfo.put(serverInfo, new BalanceInfo(numToOffload, (-1) * numTaken)); } // Walk down least loaded, filling each to the min int serversUnderloaded = 0; // number of servers that get new regions int neededRegions = 0; // number of regions needed to bring all up to min for (Map.Entry<HServerInfo, List<HRegionInfo>> server : serversByLoad.entrySet()) { int regionCount = server.getKey().getLoad().getNumberOfRegions(); if (regionCount >= min) { break; } serversUnderloaded++; int numToTake = min - regionCount; int numTaken = 0; while (numTaken < numToTake && regionidx < regionsToMove.size()) { regionsToMove.get(regionidx).setDestination(server.getKey()); numTaken++; regionidx++; } serverBalanceInfo.put(server.getKey(), new BalanceInfo(0, numTaken)); // If we still want to take some, increment needed if (numTaken < numToTake) { neededRegions += (numToTake - numTaken); } } // If none needed to fill all to min and none left to drain all to max, // we are done if (neededRegions == 0 && regionidx == regionsToMove.size()) { long endTime = System.currentTimeMillis(); LOG.info("Calculated a load balance in " + (endTime - startTime) + "ms. " + "Moving " + regionsToMove.size() + " regions off of " + serversOverloaded + " overloaded servers onto " + serversUnderloaded + " less loaded servers"); return regionsToMove; } // Need to do a second pass. // Either more regions to assign out or servers that are still underloaded // If we need more to fill min, grab one from each most loaded until enough if (neededRegions != 0) { // Walk down most loaded, grabbing one from each until we get enough for (Map.Entry<HServerInfo, List<HRegionInfo>> server : serversByLoad.descendingMap().entrySet()) { BalanceInfo balanceInfo = serverBalanceInfo.get(server.getKey()); int idx = balanceInfo == null ? 0 : balanceInfo.getNextRegionForUnload(); if (idx >= server.getValue().size()) break; HRegionInfo region = server.getValue().get(idx); if (region.isMetaRegion()) continue; // Don't move meta regions. regionsToMove.add(new RegionPlan(region, server.getKey(), null)); if (--neededRegions == 0) { // No more regions needed, done shedding break; } } } // Now we have a set of regions that must be all assigned out // Assign each underloaded up to the min, then if leftovers, assign to max // Walk down least loaded, assigning to each to fill up to min for (Map.Entry<HServerInfo, List<HRegionInfo>> server : serversByLoad.entrySet()) { int regionCount = server.getKey().getLoad().getNumberOfRegions(); if (regionCount >= min) break; BalanceInfo balanceInfo = serverBalanceInfo.get(server.getKey()); if (balanceInfo != null) { regionCount += balanceInfo.getNumRegionsAdded(); } if (regionCount >= min) { continue; } int numToTake = min - regionCount; int numTaken = 0; while (numTaken < numToTake && regionidx < regionsToMove.size()) { regionsToMove.get(regionidx).setDestination(server.getKey()); numTaken++; regionidx++; } } // If we still have regions to dish out, assign underloaded to max if (regionidx != regionsToMove.size()) { for (Map.Entry<HServerInfo, List<HRegionInfo>> server : serversByLoad.entrySet()) { int regionCount = server.getKey().getLoad().getNumberOfRegions(); if (regionCount >= max) { break; } regionsToMove.get(regionidx).setDestination(server.getKey()); regionidx++; if (regionidx == regionsToMove.size()) { break; } } } long endTime = System.currentTimeMillis(); if (regionidx != regionsToMove.size() || neededRegions != 0) { // Emit data so can diagnose how balancer went astray. LOG.warn("regionidx=" + regionidx + ", regionsToMove=" + regionsToMove.size() + ", numServers=" + numServers + ", serversOverloaded=" + serversOverloaded + ", serversUnderloaded=" + serversUnderloaded); StringBuilder sb = new StringBuilder(); for (Map.Entry<HServerInfo, List<HRegionInfo>> e : clusterState.entrySet()) { if (sb.length() > 0) sb.append(", "); sb.append(e.getKey().getServerName()); sb.append(" "); sb.append(e.getValue().size()); } LOG.warn("Input " + sb.toString()); } // All done! LOG.info("Calculated a load balance in " + (endTime - startTime) + "ms. " + "Moving " + regionsToMove.size() + " regions off of " + serversOverloaded + " overloaded servers onto " + serversUnderloaded + " less loaded servers"); return regionsToMove; }
From source file:com.datatorrent.stram.cli.DTCli.java
String[] getLaunchAppPackageArgs(AppPackage ap, ConfigPackage cp, LaunchCommandLineInfo commandLineInfo,
ConsoleReader reader) throws Exception {
String matchAppName = null;/*from w w w . j a v a 2 s .co m*/
if (commandLineInfo.args.length > 1) {
matchAppName = commandLineInfo.args[1];
}
List<AppInfo> applications = new ArrayList<AppInfo>(ap.getApplications());
if (matchAppName != null) {
Iterator<AppInfo> it = applications.iterator();
while (it.hasNext()) {
AppInfo ai = it.next();
if ((commandLineInfo.exactMatch && !ai.name.equals(matchAppName))
|| !ai.name.toLowerCase().matches(".*" + matchAppName.toLowerCase() + ".*")) {
it.remove();
}
}
}
AppInfo selectedApp = null;
if (applications.isEmpty()) {
throw new CliException("No applications in Application Package"
+ (matchAppName != null ? " matching \"" + matchAppName + "\"" : ""));
} else if (applications.size() == 1) {
selectedApp = applications.get(0);
} else {
//Store the appNames sorted in alphabetical order and their position in matchingAppFactories list
TreeMap<String, Integer> appNamesInAlphabeticalOrder = new TreeMap<String, Integer>();
// Display matching applications
for (int i = 0; i < applications.size(); i++) {
String appName = applications.get(i).name;
appNamesInAlphabeticalOrder.put(appName, i);
}
//Create a mapping between the app display number and original index at matchingAppFactories
int index = 1;
HashMap<Integer, Integer> displayIndexToOriginalUnsortedIndexMap = new HashMap<Integer, Integer>();
for (Map.Entry<String, Integer> entry : appNamesInAlphabeticalOrder.entrySet()) {
//Map display number of the app to original unsorted index
displayIndexToOriginalUnsortedIndexMap.put(index, entry.getValue());
//Display the app names
System.out.printf("%3d. %s\n", index++, entry.getKey());
}
// Exit if not in interactive mode
if (!consolePresent) {
throw new CliException(
"More than one application in Application Package match '" + matchAppName + "'");
} else {
boolean useHistory = reader.isHistoryEnabled();
reader.setHistoryEnabled(false);
History previousHistory = reader.getHistory();
History dummyHistory = new MemoryHistory();
reader.setHistory(dummyHistory);
List<Completer> completers = new ArrayList<Completer>(reader.getCompleters());
for (Completer c : completers) {
reader.removeCompleter(c);
}
reader.setHandleUserInterrupt(true);
String optionLine;
try {
optionLine = reader.readLine("Choose application: ");
} finally {
reader.setHandleUserInterrupt(false);
reader.setHistoryEnabled(useHistory);
reader.setHistory(previousHistory);
for (Completer c : completers) {
reader.addCompleter(c);
}
}
try {
int option = Integer.parseInt(optionLine);
if (0 < option && option <= applications.size()) {
int appIndex = displayIndexToOriginalUnsortedIndexMap.get(option);
selectedApp = applications.get(appIndex);
}
} catch (Exception ex) {
// ignore
}
}
}
if (selectedApp == null) {
throw new CliException("No application selected");
}
DTConfiguration launchProperties = getLaunchAppPackageProperties(ap, cp, commandLineInfo, selectedApp.name);
String appFile = ap.tempDirectory() + "/app/" + selectedApp.file;
List<String> launchArgs = new ArrayList<String>();
launchArgs.add("launch");
launchArgs.add("-exactMatch");
List<String> absClassPath = new ArrayList<String>(ap.getClassPath());
for (int i = 0; i < absClassPath.size(); i++) {
String path = absClassPath.get(i);
if (!path.startsWith("/")) {
absClassPath.set(i, ap.tempDirectory() + "/" + path);
}
}
if (cp != null) {
StringBuilder files = new StringBuilder();
for (String file : cp.getClassPath()) {
if (files.length() != 0) {
files.append(',');
}
files.append(cp.tempDirectory()).append(File.separatorChar).append(file);
}
if (!StringUtils.isBlank(files.toString())) {
if (commandLineInfo.libjars != null) {
commandLineInfo.libjars = files.toString() + "," + commandLineInfo.libjars;
} else {
commandLineInfo.libjars = files.toString();
}
}
files.setLength(0);
for (String file : cp.getFiles()) {
if (files.length() != 0) {
files.append(',');
}
files.append(cp.tempDirectory()).append(File.separatorChar).append(file);
}
if (!StringUtils.isBlank(files.toString())) {
if (commandLineInfo.files != null) {
commandLineInfo.files = files.toString() + "," + commandLineInfo.files;
} else {
commandLineInfo.files = files.toString();
}
}
}
StringBuilder libjarsVal = new StringBuilder();
if (!absClassPath.isEmpty() || commandLineInfo.libjars != null) {
if (!absClassPath.isEmpty()) {
libjarsVal.append(org.apache.commons.lang3.StringUtils.join(absClassPath, ','));
}
if (commandLineInfo.libjars != null) {
if (libjarsVal.length() > 0) {
libjarsVal.append(",");
}
libjarsVal.append(commandLineInfo.libjars);
}
}
if (appFile.endsWith(".json") || appFile.endsWith(".properties")) {
if (libjarsVal.length() > 0) {
libjarsVal.append(",");
}
libjarsVal.append(ap.tempDirectory()).append("/app/*.jar");
}
if (libjarsVal.length() > 0) {
launchArgs.add("-libjars");
launchArgs.add(libjarsVal.toString());
}
File launchPropertiesFile = new File(ap.tempDirectory(), "launch.xml");
launchProperties.writeToFile(launchPropertiesFile, "");
launchArgs.add("-conf");
launchArgs.add(launchPropertiesFile.getCanonicalPath());
if (commandLineInfo.localMode) {
launchArgs.add("-local");
}
if (commandLineInfo.archives != null) {
launchArgs.add("-archives");
launchArgs.add(commandLineInfo.archives);
}
if (commandLineInfo.files != null) {
launchArgs.add("-files");
launchArgs.add(commandLineInfo.files);
}
if (commandLineInfo.origAppId != null) {
launchArgs.add("-originalAppId");
launchArgs.add(commandLineInfo.origAppId);
}
if (commandLineInfo.queue != null) {
launchArgs.add("-queue");
launchArgs.add(commandLineInfo.queue);
}
launchArgs.add(appFile);
if (!appFile.endsWith(".json") && !appFile.endsWith(".properties")) {
launchArgs.add(selectedApp.name);
}
LOG.debug("Launch command: {}", StringUtils.join(launchArgs, " "));
return launchArgs.toArray(new String[] {});
}
From source file:com.datatorrent.stram.cli.ApexCli.java
String[] getLaunchAppPackageArgs(AppPackage ap, ConfigPackage cp, LaunchCommandLineInfo commandLineInfo,
ConsoleReader reader) throws Exception {
String matchAppName = null;/*from ww w . j a va 2 s . c om*/
if (commandLineInfo.args.length > 1) {
matchAppName = commandLineInfo.args[1];
}
List<AppInfo> applications = new ArrayList<>(
getAppsFromPackageAndConfig(ap, cp, commandLineInfo.useConfigApps));
if (matchAppName != null) {
Iterator<AppInfo> it = applications.iterator();
while (it.hasNext()) {
AppInfo ai = it.next();
if ((commandLineInfo.exactMatch && !ai.name.equals(matchAppName))
|| !ai.name.toLowerCase().matches(".*" + matchAppName.toLowerCase() + ".*")) {
it.remove();
}
}
}
AppInfo selectedApp = null;
if (applications.isEmpty()) {
throw new CliException("No applications in Application Package"
+ (matchAppName != null ? " matching \"" + matchAppName + "\"" : ""));
} else if (applications.size() == 1) {
selectedApp = applications.get(0);
} else {
//Store the appNames sorted in alphabetical order and their position in matchingAppFactories list
TreeMap<String, Integer> appNamesInAlphabeticalOrder = new TreeMap<>();
// Display matching applications
for (int i = 0; i < applications.size(); i++) {
String appName = applications.get(i).name;
appNamesInAlphabeticalOrder.put(appName, i);
}
//Create a mapping between the app display number and original index at matchingAppFactories
int index = 1;
HashMap<Integer, Integer> displayIndexToOriginalUnsortedIndexMap = new HashMap<>();
for (Map.Entry<String, Integer> entry : appNamesInAlphabeticalOrder.entrySet()) {
//Map display number of the app to original unsorted index
displayIndexToOriginalUnsortedIndexMap.put(index, entry.getValue());
//Display the app names
System.out.printf("%3d. %s\n", index++, entry.getKey());
}
// Exit if not in interactive mode
if (!consolePresent) {
throw new CliException(
"More than one application in Application Package match '" + matchAppName + "'");
} else {
boolean useHistory = reader.isHistoryEnabled();
reader.setHistoryEnabled(false);
History previousHistory = reader.getHistory();
History dummyHistory = new MemoryHistory();
reader.setHistory(dummyHistory);
List<Completer> completers = new ArrayList<>(reader.getCompleters());
for (Completer c : completers) {
reader.removeCompleter(c);
}
reader.setHandleUserInterrupt(true);
String optionLine;
try {
optionLine = reader.readLine("Choose application: ");
} finally {
reader.setHandleUserInterrupt(false);
reader.setHistoryEnabled(useHistory);
reader.setHistory(previousHistory);
for (Completer c : completers) {
reader.addCompleter(c);
}
}
try {
int option = Integer.parseInt(optionLine);
if (0 < option && option <= applications.size()) {
int appIndex = displayIndexToOriginalUnsortedIndexMap.get(option);
selectedApp = applications.get(appIndex);
}
} catch (Exception ex) {
// ignore
}
}
}
if (selectedApp == null) {
throw new CliException("No application selected");
}
DTConfiguration launchProperties = getLaunchAppPackageProperties(ap, cp, commandLineInfo, selectedApp.name);
String appFile = ap.tempDirectory() + "/app/" + selectedApp.file;
List<String> launchArgs = new ArrayList<>();
launchArgs.add("launch");
launchArgs.add("-exactMatch");
List<String> absClassPath = new ArrayList<>(ap.getClassPath());
for (int i = 0; i < absClassPath.size(); i++) {
String path = absClassPath.get(i);
if (!path.startsWith("/")) {
absClassPath.set(i, ap.tempDirectory() + "/" + path);
}
}
if (cp != null) {
StringBuilder files = new StringBuilder();
for (String file : cp.getClassPath()) {
if (files.length() != 0) {
files.append(',');
}
files.append(cp.tempDirectory()).append(File.separatorChar).append(file);
}
if (!StringUtils.isBlank(files.toString())) {
if (commandLineInfo.libjars != null) {
commandLineInfo.libjars = files.toString() + "," + commandLineInfo.libjars;
} else {
commandLineInfo.libjars = files.toString();
}
}
files.setLength(0);
for (String file : cp.getFiles()) {
if (files.length() != 0) {
files.append(',');
}
files.append(cp.tempDirectory()).append(File.separatorChar).append(file);
}
if (!StringUtils.isBlank(files.toString())) {
if (commandLineInfo.files != null) {
commandLineInfo.files = files.toString() + "," + commandLineInfo.files;
} else {
commandLineInfo.files = files.toString();
}
}
}
StringBuilder libjarsVal = new StringBuilder();
if (!absClassPath.isEmpty() || commandLineInfo.libjars != null) {
if (!absClassPath.isEmpty()) {
libjarsVal.append(org.apache.commons.lang3.StringUtils.join(absClassPath, ','));
}
if (commandLineInfo.libjars != null) {
if (libjarsVal.length() > 0) {
libjarsVal.append(",");
}
libjarsVal.append(commandLineInfo.libjars);
}
}
if (appFile.endsWith(".json") || appFile.endsWith(".properties")) {
if (libjarsVal.length() > 0) {
libjarsVal.append(",");
}
libjarsVal.append(ap.tempDirectory()).append("/app/*.jar");
}
if (libjarsVal.length() > 0) {
launchArgs.add("-libjars");
launchArgs.add(libjarsVal.toString());
}
File launchPropertiesFile = new File(ap.tempDirectory(), "launch.xml");
launchProperties.writeToFile(launchPropertiesFile, "");
launchArgs.add("-conf");
launchArgs.add(launchPropertiesFile.getCanonicalPath());
if (commandLineInfo.localMode) {
launchArgs.add("-local");
}
if (commandLineInfo.archives != null) {
launchArgs.add("-archives");
launchArgs.add(commandLineInfo.archives);
}
if (commandLineInfo.files != null) {
launchArgs.add("-files");
launchArgs.add(commandLineInfo.files);
}
if (commandLineInfo.origAppId != null) {
launchArgs.add("-originalAppId");
launchArgs.add(commandLineInfo.origAppId);
}
if (commandLineInfo.queue != null) {
launchArgs.add("-queue");
launchArgs.add(commandLineInfo.queue);
}
if (commandLineInfo.tags != null) {
launchArgs.add("-tags");
launchArgs.add(commandLineInfo.tags);
}
launchArgs.add(appFile);
if (!appFile.endsWith(".json") && !appFile.endsWith(".properties")) {
launchArgs.add(selectedApp.name);
}
LOG.debug("Launch command: {}", StringUtils.join(launchArgs, " "));
return launchArgs.toArray(new String[] {});
}
From source file:org.apache.rya.rdftriplestore.inference.InferenceEngine.java
public void refreshGraph() throws InferenceEngineException { try {// w w w . jav a 2 s. c o m CloseableIteration<Statement, QueryEvaluationException> iter; //get all subclassof Graph graph = TinkerGraph.open(); addPredicateEdges(RDFS.SUBCLASSOF, Direction.OUT, graph, RDFS.SUBCLASSOF.stringValue()); //equivalentClass is the same as subClassOf both ways addPredicateEdges(OWL.EQUIVALENTCLASS, Direction.BOTH, graph, RDFS.SUBCLASSOF.stringValue()); // Add unions to the subclass graph: if c owl:unionOf LIST(c1, c2, ... cn), then any // instances of c1, c2, ... or cn are also instances of c, meaning c is a superclass // of all the rest. // (In principle, an instance of c is likewise implied to be at least one of the other // types, but this fact is ignored for now to avoid nondeterministic reasoning.) iter = RyaDAOHelper.query(ryaDAO, null, OWL.UNIONOF, null, conf); try { while (iter.hasNext()) { final Statement st = iter.next(); final Value unionType = st.getSubject(); // Traverse the list of types constituting the union Value current = st.getObject(); while (current instanceof Resource && !RDF.NIL.equals(current)) { final Resource listNode = (Resource) current; CloseableIteration<Statement, QueryEvaluationException> listIter = RyaDAOHelper .query(ryaDAO, listNode, RDF.FIRST, null, conf); try { if (listIter.hasNext()) { final Statement firstStatement = listIter.next(); if (firstStatement.getObject() instanceof Resource) { final Resource subclass = (Resource) firstStatement.getObject(); final Statement subclassStatement = VF.createStatement(subclass, RDFS.SUBCLASSOF, unionType); addStatementEdge(graph, RDFS.SUBCLASSOF.stringValue(), subclassStatement); } } } finally { listIter.close(); } listIter = RyaDAOHelper.query(ryaDAO, listNode, RDF.REST, null, conf); try { if (listIter.hasNext()) { current = listIter.next().getObject(); } else { current = RDF.NIL; } } finally { listIter.close(); } } } } finally { if (iter != null) { iter.close(); } } subClassOfGraph = graph; //TODO: Should this be synchronized? graph = TinkerGraph.open(); addPredicateEdges(RDFS.SUBPROPERTYOF, Direction.OUT, graph, RDFS.SUBPROPERTYOF.stringValue()); //equiv property really is the same as a subPropertyOf both ways addPredicateEdges(OWL.EQUIVALENTPROPERTY, Direction.BOTH, graph, RDFS.SUBPROPERTYOF.stringValue()); subPropertyOfGraph = graph; //TODO: Should this be synchronized? refreshIntersectionOf(); refreshOneOf(); symmetricPropertySet = fetchInstances(OWL.SYMMETRICPROPERTY); transitivePropertySet = fetchInstances(OWL.TRANSITIVEPROPERTY); reflexivePropertySet = fetchInstances(REFLEXIVE_PROPERTY); iter = RyaDAOHelper.query(ryaDAO, null, OWL.INVERSEOF, null, conf); final Map<URI, URI> invProp = new HashMap<>(); try { while (iter.hasNext()) { final Statement st = iter.next(); invProp.put((URI) st.getSubject(), (URI) st.getObject()); invProp.put((URI) st.getObject(), (URI) st.getSubject()); } } finally { if (iter != null) { iter.close(); } } inverseOfMap = invProp; iter = RyaDAOHelper.query(ryaDAO, null, VF.createURI("http://www.w3.org/2002/07/owl#propertyChainAxiom"), null, conf); final Map<URI, URI> propertyChainPropertiesToBNodes = new HashMap<>(); propertyChainPropertyToChain = new HashMap<>(); try { while (iter.hasNext()) { final Statement st = iter.next(); propertyChainPropertiesToBNodes.put((URI) st.getSubject(), (URI) st.getObject()); } } finally { if (iter != null) { iter.close(); } } // now for each property chain bNode, get the indexed list of properties associated with that chain for (final URI propertyChainProperty : propertyChainPropertiesToBNodes.keySet()) { final URI bNode = propertyChainPropertiesToBNodes.get(propertyChainProperty); // query for the list of indexed properties iter = RyaDAOHelper.query(ryaDAO, bNode, VF.createURI("http://www.w3.org/2000/10/swap/list#index"), null, conf); final TreeMap<Integer, URI> orderedProperties = new TreeMap<>(); // TODO refactor this. Wish I could execute sparql try { while (iter.hasNext()) { final Statement st = iter.next(); final String indexedElement = st.getObject().stringValue(); log.info(indexedElement); CloseableIteration<Statement, QueryEvaluationException> iter2 = RyaDAOHelper.query(ryaDAO, VF.createURI(st.getObject().stringValue()), RDF.FIRST, null, conf); String integerValue = ""; Value anonPropNode = null; Value propURI = null; if (iter2 != null) { while (iter2.hasNext()) { final Statement iter2Statement = iter2.next(); integerValue = iter2Statement.getObject().stringValue(); break; } iter2.close(); } iter2 = RyaDAOHelper.query(ryaDAO, VF.createURI(st.getObject().stringValue()), RDF.REST, null, conf); if (iter2 != null) { while (iter2.hasNext()) { final Statement iter2Statement = iter2.next(); anonPropNode = iter2Statement.getObject(); break; } iter2.close(); if (anonPropNode != null) { iter2 = RyaDAOHelper.query(ryaDAO, VF.createURI(anonPropNode.stringValue()), RDF.FIRST, null, conf); while (iter2.hasNext()) { final Statement iter2Statement = iter2.next(); propURI = iter2Statement.getObject(); break; } iter2.close(); } } if (!integerValue.isEmpty() && propURI != null) { try { final int indexValue = Integer.parseInt(integerValue); final URI chainPropURI = VF.createURI(propURI.stringValue()); orderedProperties.put(indexValue, chainPropURI); } catch (final Exception ex) { // TODO log an error here } } } } finally { if (iter != null) { iter.close(); } } final List<URI> properties = new ArrayList<>(); for (final Map.Entry<Integer, URI> entry : orderedProperties.entrySet()) { properties.add(entry.getValue()); } propertyChainPropertyToChain.put(propertyChainProperty, properties); } // could also be represented as a list of properties (some of which may be blank nodes) for (final URI propertyChainProperty : propertyChainPropertiesToBNodes.keySet()) { final List<URI> existingChain = propertyChainPropertyToChain.get(propertyChainProperty); // if we didn't get a chain, try to get it through following the collection if ((existingChain == null) || existingChain.isEmpty()) { CloseableIteration<Statement, QueryEvaluationException> iter2 = RyaDAOHelper.query(ryaDAO, propertyChainPropertiesToBNodes.get(propertyChainProperty), RDF.FIRST, null, conf); final List<URI> properties = new ArrayList<>(); URI previousBNode = propertyChainPropertiesToBNodes.get(propertyChainProperty); if (iter2.hasNext()) { Statement iter2Statement = iter2.next(); Value currentPropValue = iter2Statement.getObject(); while ((currentPropValue != null) && (!currentPropValue.stringValue().equalsIgnoreCase(RDF.NIL.stringValue()))) { if (currentPropValue instanceof URI) { iter2 = RyaDAOHelper.query(ryaDAO, VF.createURI(currentPropValue.stringValue()), RDF.FIRST, null, conf); if (iter2.hasNext()) { iter2Statement = iter2.next(); if (iter2Statement.getObject() instanceof URI) { properties.add((URI) iter2Statement.getObject()); } } // otherwise see if there is an inverse declaration else { iter2 = RyaDAOHelper.query(ryaDAO, VF.createURI(currentPropValue.stringValue()), OWL.INVERSEOF, null, conf); if (iter2.hasNext()) { iter2Statement = iter2.next(); if (iter2Statement.getObject() instanceof URI) { properties.add(new InverseURI((URI) iter2Statement.getObject())); } } } // get the next prop pointer iter2 = RyaDAOHelper.query(ryaDAO, previousBNode, RDF.REST, null, conf); if (iter2.hasNext()) { iter2Statement = iter2.next(); previousBNode = (URI) currentPropValue; currentPropValue = iter2Statement.getObject(); } else { currentPropValue = null; } } else { currentPropValue = null; } } propertyChainPropertyToChain.put(propertyChainProperty, properties); } } } refreshDomainRange(); refreshPropertyRestrictions(); } catch (final QueryEvaluationException e) { throw new InferenceEngineException(e); } }
From source file:AnalysisModule.DataAnalysis.java
protected void bitmapAnalyse(List<Scenario> lstScenario) throws Exception { BitMap bitMap;/*from www. j ava2s. com*/ String bmpDir; int traffic; int numberOfFiles; FileInputStream fin = null; ObjectInputStream ois = null; for (Scenario scenario : lstScenario) { for (Topology topology : scenario.lstTopology) { for (Instance instance : topology.getLstInstance()) { instance.trafficMatrix = new double[topology.getNumberOfSwitches()][topology .getNumberOfSwitches()]; } } } for (Scenario scenario : lstScenario) { for (Topology topology : scenario.lstTopology) { System.out.println("Analisando Topologia: " + topology.getIdTopology()); for (Instance instance : topology.getLstInstance()) { System.out.println(" Instancia: " + instance.getId()); for (int i = 0; i < topology.getNumberOfSwitches(); i++) { for (int j = 0; j < topology.getNumberOfSwitches(); j++) { switch (instance.type) { case BITMAP: { if (j <= i) { break; } TreeMap<Long, BitMap> sourceBitMapTree = new TreeMap<>(); TreeMap<Long, BitMap> destinationBitMapTree = new TreeMap<>(); bmpDir = getSrcDir(instance, i); numberOfFiles = new File(bmpDir).listFiles().length - 1; int firstBitmap = -2; for (int numberOfBmp = 0; numberOfBmp < numberOfFiles; numberOfBmp++) { fin = new FileInputStream(bmpDir + "BitMap" + numberOfBmp + ".bmp"); ois = new ObjectInputStream(fin); bitMap = (BitMap) ois.readObject(); if ((bitMap.getStartEpoch() - 21600000000L > scenario.startTime) && (bitMap.getStartEpoch() - 21600000000L < scenario.endTime)) { if (firstBitmap == -2) { firstBitmap = numberOfBmp - 1; } sourceBitMapTree.put(bitMap.getStartEpoch() - 21600000000L, bitMap); } ois.close(); fin.close(); } //Add the first bitmap in the Measurement Interval if (firstBitmap >= 0) { fin = new FileInputStream(bmpDir + "BitMap" + firstBitmap + ".bmp"); ois = new ObjectInputStream(fin); bitMap = (BitMap) ois.readObject(); sourceBitMapTree.put(bitMap.getStartEpoch() - 21600000000L, bitMap); ois.close(); fin.close(); } bmpDir = getDestDir(instance, j); numberOfFiles = new File(bmpDir).listFiles().length - 1; firstBitmap = -2; for (int numberOfBmp = 0; numberOfBmp < numberOfFiles; numberOfBmp++) { fin = new FileInputStream(bmpDir + "BitMap" + numberOfBmp + ".bmp"); ois = new ObjectInputStream(fin); bitMap = (BitMap) ois.readObject(); if ((bitMap.getStartEpoch() - 21600000000L > scenario.startTime) && (bitMap.getStartEpoch() - 21600000000L < scenario.endTime)) { if (firstBitmap == -2) { firstBitmap = numberOfBmp - 1; } destinationBitMapTree.put(bitMap.getStartEpoch() - 21600000000L, bitMap); } ois.close(); fin.close(); } //Add the first bitmap in the Measurement Interval if (firstBitmap >= 0) { fin = new FileInputStream(bmpDir + "BitMap" + firstBitmap + ".bmp"); ois = new ObjectInputStream(fin); bitMap = (BitMap) ois.readObject(); destinationBitMapTree.put(bitMap.getStartEpoch() - 21600000000L, bitMap); ois.close(); fin.close(); } //Estimation int k1 = sourceBitMapTree.size(); int k2 = destinationBitMapTree.size(); Collection sourceEntrySet = sourceBitMapTree.entrySet(); Iterator sourceEntries = sourceEntrySet.iterator(); for (int q = 0; q < k1; q++) { Map.Entry entrySrc = (Map.Entry) sourceEntries.next(); BitMap bmpSrc = (BitMap) entrySrc.getValue(); Collection destinationEntrySet = destinationBitMapTree.entrySet(); Iterator destinationEntries = destinationEntrySet.iterator(); for (int r = 0; r < k2; r++) { Map.Entry entryDst = (Map.Entry) destinationEntries.next(); BitMap bmpDst = (BitMap) entryDst.getValue(); boolean overlap = bmpSrc.getStartEpoch() <= bmpDst.getEndEpoch() && bmpSrc.getEndEpoch() >= bmpDst.getStartEpoch(); if (overlap) { double sourceDTr = instance.getBitMapSize() * Math.log(((double) instance.getBitMapSize()) / (instance.getBitMapSize() - bmpSrc.occupancy())); double destinationDTr = instance.getBitMapSize() * Math.log(((double) instance.getBitMapSize()) / (instance.getBitMapSize() - bmpDst.occupancy())); BitSet orSrcDst = (BitSet) bmpSrc.getBitSet().clone(); //BitSet andSrcDst = (BitSet) bmpSrc.getBitSet().clone(); orSrcDst.or(bmpDst.getBitSet()); //andSrcDst.and(bmpDst.getBitSet()); double orDTr = instance.getBitMapSize() * Math.log(((double) instance.getBitMapSize()) / (instance.getBitMapSize() - orSrcDst.cardinality())); //double andDTr = instance.getBitMapSize() * Math.log(((double) instance.getBitMapSize()) / (instance.getBitMapSize() - andSrcDst.cardinality())); double estimation = 0D; if (Double.isFinite(orDTr)) { estimation = sourceDTr + destinationDTr - orDTr; //estimation = (bmpSrc.getNumberOfPackets()/sourceDTr) * estimation; //estimation = andDTr; } instance.trafficMatrix[i][j] += estimation; } } } break; } case COUNTER_ARRAY: { traffic = 0; traffic += instance.networkSwitch.get(i).arrayCounter[i][j]; //traffic += instance.networkSwitch.get(i).arrayCounter[j][i]; instance.doCalculateMatrixElem(i, j, topology, traffic); break; } case OPT_COUNTER_ARRAY: { traffic = 0; for (Integer node : topology.getPathNodes(i, j)) { if (instance.networkSwitch.get(node).isObserver) { traffic += instance.networkSwitch.get(node).arrayCounter[i][j]; //traffic += instance.networkSwitch.get(node).arrayCounter[j][i]; break; } } instance.doCalculateMatrixElem(i, j, topology, traffic); break; } } } } } } } }
From source file:org.biomart.configurator.controller.MartController.java
/** * @param fksToBeDropped//from ww w.java2s. c om * @param dmd * @param schema * @param catalog * @param stepSize * @throws SQLException * @throws DataModelException */ public void synchroniseKeysUsingDMD(final SourceSchema ss, final Collection<ForeignKey> fksToBeDropped, final DatabaseMetaData dmd, final String schema, final String catalog) throws SQLException, DataModelException { Log.debug("Running DMD key synchronisation"); // Loop through all the tables in the database, which is the same // as looping through all the primary keys. Log.debug("Finding tables"); for (final Iterator<Table> i = ss.getTables().iterator(); i.hasNext();) { // Obtain the table and its primary key. final SourceTable pkTable = (SourceTable) i.next(); final PrimaryKey pk = pkTable.getPrimaryKey(); // Skip all tables which have no primary key. if (pk == null) continue; Log.debug("Processing primary key " + pk); // Make a list of relations that already exist in this schema, // from some previous run. Any relations that are left in this // list by the end of the loop for this table no longer exist in // the database, and will be dropped. final Collection<Relation> relationsToBeDropped = new TreeSet<Relation>(pk.getRelations()); // Tree for // order // Identify all foreign keys in the database metadata that refer // to the current primary key. Log.debug("Finding referring foreign keys"); String searchCatalog = catalog; String searchSchema = schema; final ResultSet dbTblFKCols = dmd.getExportedKeys(searchCatalog, searchSchema, pkTable.getName()); // Loop through the results. There will be one result row per // column per key, so we need to build up a set of key columns // in a map. // The map keys represent the column position within a key. Each // map value is a list of columns. In essence the map is a 2-D // representation of the foreign keys which refer to this PK, // with the keys of the map (Y-axis) representing the column // position in the FK, and the values of the map (X-axis) // representing each individual FK. In all cases, FK columns are // assumed to be in the same order as the PK columns. The map is // sorted by key column position. // An assumption is made that the query will return columns from // the FK in the same order as all other FKs, ie. all column 1s // will be returned before any 2s, and then all 2s will be // returned // in the same order as the 1s they are associated with, etc. final TreeMap<Short, List<Column>> dbFKs = new TreeMap<Short, List<Column>>(); while (dbTblFKCols.next()) { final String fkTblName = dbTblFKCols.getString("FKTABLE_NAME"); final String fkColName = dbTblFKCols.getString("FKCOLUMN_NAME"); final Short fkColSeq = new Short(dbTblFKCols.getShort("KEY_SEQ")); if (fkTblName != null && fkTblName.contains("$")) { // exclude ORACLE's temporary tables (unlikely to be // found here though) continue; } // Note the column. if (!dbFKs.containsKey(fkColSeq)) dbFKs.put(fkColSeq, new ArrayList<Column>()); // In some dbs, FKs can be invalid, so we need to check // them. final Table fkTbl = ss.getTableByName(fkTblName); if (fkTbl != null) { final Column fkCol = (Column) fkTbl.getColumnByName(fkColName); if (fkCol != null) (dbFKs.get(fkColSeq)).add(fkCol); } } dbTblFKCols.close(); // Sort foreign keys by name (case insensitive) for (List<Column> columnList : dbFKs.values()) { Collections.sort(columnList); } // Only construct FKs if we actually found any. if (!dbFKs.isEmpty()) { // Identify the sequence of the first column, which may be 0 // or 1, depending on database implementation. final int firstColSeq = ((Short) dbFKs.firstKey()).intValue(); // How many columns are in the PK? final int pkColCount = pkTable.getPrimaryKey().getColumns().size(); // How many FKs do we have? final int fkCount = dbFKs.get(dbFKs.firstKey()).size(); // Loop through the FKs, and construct each one at a time. for (int j = 0; j < fkCount; j++) { // Set up an array to hold the FK columns. final List<Column> candidateFKColumns = new ArrayList<Column>(); // For each FK column name, look up the actual column in // the table. for (final Iterator<Map.Entry<Short, List<Column>>> k = dbFKs.entrySet().iterator(); k .hasNext();) { final Map.Entry<Short, List<Column>> entry = k.next(); final Short keySeq = (Short) entry.getKey(); // Convert the db-specific column index to a // 0-indexed figure for the array of fk columns. final int fkColSeq = keySeq.intValue() - firstColSeq; candidateFKColumns.add((Column) (entry.getValue()).get(j)); } // Create a template foreign key based around the set // of candidate columns we found. ForeignKey fkObject; try { List<Column> columns = new ArrayList<Column>(); for (int k = 0; k < candidateFKColumns.size(); k++) { columns.add(candidateFKColumns.get(k)); } fkObject = new ForeignKey(columns); // new KeyController(fkObject); } catch (final Throwable t) { throw new BioMartError(t); } final Table fkTable = fkObject.getTable(); // If any FK already exists on the target table with the // same columns in the same order, then reuse it. boolean fkAlreadyExists = false; for (final Iterator<ForeignKey> f = fkTable.getForeignKeys().iterator(); f.hasNext() && !fkAlreadyExists;) { final ForeignKey candidateFK = f.next(); if (candidateFK.equals(fkObject)) { // Found one. Reuse it! fkObject = candidateFK; // Update the status to indicate that the FK is // backed by the database, if previously it was // handmade. if (fkObject.getStatus().equals(ComponentStatus.HANDMADE)) fkObject.setStatus(ComponentStatus.INFERRED); // Remove the FK from the list to be dropped // later, as it definitely exists now. fksToBeDropped.remove(candidateFK); // Flag the key as existing. fkAlreadyExists = true; } } // Has the key been reused, or is it a new one? if (!fkAlreadyExists) try { fkTable.getForeignKeys().add(fkObject); // fkTable.getForeignKeys().add(fk); } catch (final Throwable t) { throw new BioMartError(t); } // Work out whether the relation from the FK to // the PK should be 1:M or 1:1. The rule is that // it will be 1:M in all cases except where the // FK table has a PK with identical columns to // the FK, in which case it is 1:1, as the FK // is unique. Cardinality card = Cardinality.MANY_A; final PrimaryKey fkPK = fkTable.getPrimaryKey(); if (fkPK != null && fkObject.getColumns().equals(fkPK.getColumns())) card = Cardinality.ONE; // Check to see if it already has a relation. boolean relationExists = false; for (final Iterator<Relation> f = fkObject.getRelations().iterator(); f.hasNext();) { // Obtain the next relation. final Relation candidateRel = f.next(); // a) a relation already exists between the FK // and the PK. if (candidateRel.getOtherKey(fkObject).equals(pk)) { // If cardinality matches, make it // inferred. If doesn't match, make it // modified and update original cardinality. try { if (card.equals(candidateRel.getCardinality())) { if (!candidateRel.getStatus().equals(ComponentStatus.INFERRED_INCORRECT)) candidateRel.setStatus(ComponentStatus.INFERRED); } else { if (!candidateRel.getStatus().equals(ComponentStatus.INFERRED_INCORRECT)) candidateRel.setStatus(ComponentStatus.MODIFIED); candidateRel.setOriginalCardinality(card); } } catch (final AssociationException ae) { throw new BioMartError(ae); } // Don't drop it at the end of the loop. relationsToBeDropped.remove(candidateRel); // Say we've found it. relationExists = true; } // b) a handmade relation exists elsewhere which // should not be dropped. All other relations // elsewhere will be dropped. else if (candidateRel.getStatus().equals(ComponentStatus.HANDMADE)) // Don't drop it at the end of the loop. relationsToBeDropped.remove(candidateRel); } // If relation did not already exist, create it. if (!relationExists && !pk.equals(fkObject)) { // Establish the relation. try { new RelationSource(pk, fkObject, card); // pk.getObject().addRelation(relation); // fk.getObject().addRelation(relation); } catch (final Throwable t) { throw new BioMartError(t); } } } } // Remove any relations that we didn't find in the database (but // leave the handmade ones behind). for (final Iterator<Relation> j = relationsToBeDropped.iterator(); j.hasNext();) { final Relation r = j.next(); if (r.getStatus().equals(ComponentStatus.HANDMADE)) continue; r.getFirstKey().removeRelation(r); r.getSecondKey().removeRelation(r); } } }
From source file:org.infoglue.deliver.util.CacheController.java
public static void clearCaches(String entity, String entityId, Map<String, String> extraInformation, String[] cachesToSkip, boolean forceClear) throws Exception { Timer t = new Timer(); //t.setActive(false); if (CmsPropertyHandler.getOperatingMode().equals("3")) { long wait = 0; //while(true && !getForcedCacheEvictionMode()) while (!forceClear && !getForcedCacheEvictionMode() && RequestAnalyser.getRequestAnalyser().getNumberOfActiveRequests() > 0) { logger.info(// w ww .ja v a 2 s .c o m "Number of requests: " + RequestAnalyser.getRequestAnalyser().getNumberOfCurrentRequests() + " was more than 0 - lets wait a bit."); if (wait > 1000) { logger.warn("The clearCache method waited over " + ((wait * 10) / 1000) + " seconds but there seems to be " + RequestAnalyser.getRequestAnalyser().getNumberOfCurrentRequests() + " requests blocking all the time. Continuing anyway."); //printThreads(); break; } if (wait > 100) setForcedCacheEvictionMode(true); Thread.sleep(10); wait++; } } logger.info("clearCaches start in " + CmsPropertyHandler.getContextRootPath()); if (entity == null) { logger.info("Clearing the caches"); //synchronized(caches) //{ for (Iterator i = caches.entrySet().iterator(); i.hasNext();) { Map.Entry e = (Map.Entry) i.next(); logger.info("e:" + e.getKey()); boolean skip = false; if (cachesToSkip != null) { for (int index = 0; index < cachesToSkip.length; index++) { if (e.getKey().equals(cachesToSkip[index])) { skip = true; break; } } } if (!skip) { Object object = e.getValue(); if (object instanceof Map) { Map cacheInstance = (Map) e.getValue(); synchronized (cacheInstance) { cacheInstance.clear(); } } else { GeneralCacheAdministrator cacheInstance = (GeneralCacheAdministrator) e.getValue(); synchronized (cacheInstance) { cacheInstance.flushAll(); } } logger.info("Cleared cache:" + e.getKey()); i.remove(); } } //} } else if (entity.equalsIgnoreCase("CacheNames")) { String[] cacheNames = entityId.split(","); for (int i = 0; i < cacheNames.length; i++) { String cacheName = cacheNames[i]; CacheController.clearCache(cacheName); } } else { logger.info("Clearing some caches"); logger.info("entity:" + entity); String useSelectivePageCacheUpdateString = CmsPropertyHandler.getUseSelectivePageCacheUpdate(); boolean useSelectivePageCacheUpdate = false; if (useSelectivePageCacheUpdateString != null && useSelectivePageCacheUpdateString.equalsIgnoreCase("true")) useSelectivePageCacheUpdate = true; String operatingMode = CmsPropertyHandler.getOperatingMode(); TreeMap<String, Object> orderedCaches = new TreeMap<String, Object>(new CacheComparator()); orderedCaches.putAll(caches); /* for (String key : orderedCaches.keySet()) { System.out.println("key:" + key); } */ //t.printElapsedTime("Start cache eviction"); RequestAnalyser.getRequestAnalyser().registerComponentStatistics("Start cache eviction", t.getElapsedTime()); //t.printElapsedTime("START cachesLoop:" + entity + ":" + entityId); //synchronized(caches) //{ //for (Iterator i = caches.entrySet().iterator(); i.hasNext(); ) cachesLoop: for (Iterator i = orderedCaches.entrySet().iterator(); i.hasNext();) { RequestAnalyser.getRequestAnalyser().registerComponentStatistics("cache iteration top", t.getElapsedTime()); Map.Entry e = (Map.Entry) i.next(); logger.info("e:" + e.getKey()); boolean clear = false; boolean selectiveCacheUpdate = false; String cacheName = e.getKey().toString(); if (cachesToSkip != null) { for (int index = 0; index < cachesToSkip.length; index++) { if (cacheName.equals(cachesToSkip[index])) { continue cachesLoop; } } } //t.printElapsedTime("clearCaches 3"); if (cacheName.equalsIgnoreCase("serviceDefinitionCache") && entity.indexOf("ServiceBinding") > 0) { clear = true; } if (cacheName.equalsIgnoreCase("qualifyerListCache") && (entity.indexOf("Qualifyer") > 0 || entity.indexOf("ServiceBinding") > 0)) { clear = true; } if (cacheName.equalsIgnoreCase("availableServiceBindingCache") && entity.indexOf("AvailableServiceBinding") > 0) { clear = true; } if (cacheName.equalsIgnoreCase("categoriesCache") && entity.indexOf("Category") > 0) { clear = true; } if (cacheName.equalsIgnoreCase("repositoryCache") && entity.indexOf("Repository") > 0) { clear = true; } if (cacheName.equalsIgnoreCase("languageCache") && entity.indexOf("Language") > 0) { clear = true; } if (cacheName.equalsIgnoreCase("localeCache") && entity.indexOf("Language") > 0) { clear = true; } if ((cacheName.equalsIgnoreCase("latestSiteNodeVersionCache") || cacheName.equalsIgnoreCase("pageCacheLatestSiteNodeVersions") || cacheName.equalsIgnoreCase("pageCacheSiteNodeTypeDefinition")) && entity.indexOf("SiteNode") > 0) { clear = true; selectiveCacheUpdate = true; } if ((cacheName.equalsIgnoreCase("parentSiteNodeCache") || cacheName.equalsIgnoreCase("pageCacheParentSiteNodeCache")) && entity.indexOf("SiteNode") > 0) { clear = true; } if (cacheName.equalsIgnoreCase("NavigationCache") && (entity.indexOf("SiteNode") > 0 || entity.indexOf("Content") > 0)) { clear = true; } if (cacheName.equalsIgnoreCase("pagePathCache") && (entity.indexOf("SiteNode") > 0 || entity.indexOf("Content") > 0)) { clear = true; } if (cacheName.equalsIgnoreCase("componentEditorCache") && (entity.indexOf("SiteNode") > 0 || entity.indexOf("Content") > 0)) { clear = true; } if (cacheName.equalsIgnoreCase("componentEditorVersionIdCache") && (entity.indexOf("SiteNode") > 0 || entity.indexOf("Content") > 0)) { clear = true; } if (cacheName.equalsIgnoreCase("masterLanguageCache") && (entity.indexOf("Repository") > 0 || entity.indexOf("Language") > 0)) { clear = true; } if (cacheName.equalsIgnoreCase("parentRepository") && entity.indexOf("Repository") > 0) { clear = true; } if (cacheName.startsWith("contentAttributeCache") && (entity.indexOf("Content") > -1 || entity.indexOf("AccessRight") > 0 || entity.indexOf("SystemUser") > 0 || entity.indexOf("Role") > 0 || entity.indexOf("Group") > 0)) { clear = true; selectiveCacheUpdate = true; } if (cacheName.startsWith("metaInfoContentAttributeCache") && entity.indexOf("Content") > -1) { clear = true; selectiveCacheUpdate = true; } if (cacheName.equalsIgnoreCase("contentVersionCache") && (entity.indexOf("Content") > -1 || entity.indexOf("AccessRight") > 0 || entity.indexOf("SystemUser") > 0 || entity.indexOf("Role") > 0 || entity.indexOf("Group") > 0)) { clear = true; selectiveCacheUpdate = true; } if (cacheName.startsWith("contentVersionIdCache") && (entity.indexOf("Content") > -1 || entity.indexOf("AccessRight") > 0 || entity.indexOf("SystemUser") > 0 || entity.indexOf("Role") > 0 || entity.indexOf("Group") > 0)) { clear = true; selectiveCacheUpdate = true; } if (cacheName.equalsIgnoreCase("referencingPagesCache") && (entity.indexOf("ContentVersion") > -1 || entity.indexOf("Qualifyer") > 0)) { clear = true; } if (cacheName.equalsIgnoreCase("boundSiteNodeCache") && (entity.indexOf("ServiceBinding") > 0 || entity.indexOf("Qualifyer") > 0 || entity.indexOf("SiteNodeVersion") > 0 || entity.indexOf("SiteNodeVersion") > 0 || entity.indexOf("SiteNode") > 0 || entity.indexOf("AccessRight") > 0 || entity.indexOf("SystemUser") > 0 || entity.indexOf("Role") > 0 || entity.indexOf("Group") > 0)) { clear = true; } if (cacheName.equalsIgnoreCase("boundContentCache") && (entity.indexOf("ServiceBinding") > 0 || entity.indexOf("Qualifyer") > 0 || entity.indexOf("SiteNodeVersion") > 0 || entity.indexOf("ContentVersion") > 0 || entity.indexOf("Content") > 0 || entity.indexOf("AccessRight") > 0 || entity.indexOf("SystemUser") > 0 || entity.indexOf("Role") > 0 || entity.indexOf("Group") > 0)) { clear = true; } if (cacheName.startsWith("pageCache") && entity.indexOf("Registry") == -1) { clear = true; selectiveCacheUpdate = true; } if (cacheName.startsWith("pageCacheExtra") && entity.indexOf("Registry") == -1) { clear = true; selectiveCacheUpdate = true; } if (cacheName.equalsIgnoreCase("componentCache") && entity.indexOf("Registry") == -1) { clear = true; selectiveCacheUpdate = true; } if (cacheName.equalsIgnoreCase("componentPropertyCache") && (entity.indexOf("SiteNode") > -1 || entity.indexOf("ContentVersion") > -1 || entity.indexOf("AccessRight") > 0 || entity.indexOf("SystemUser") > 0 || entity.indexOf("Role") > 0 || entity.indexOf("Group") > 0)) { clear = true; //selectiveCacheUpdate = true; } if (cacheName.equalsIgnoreCase("componentPropertyVersionIdCache") && (entity.indexOf("SiteNode") > -1 || entity.indexOf("ContentVersion") > -1 || entity.indexOf("AccessRight") > 0 || entity.indexOf("SystemUser") > 0 || entity.indexOf("Role") > 0 || entity.indexOf("Group") > 0)) { clear = true; //selectiveCacheUpdate = true; } if (cacheName.equalsIgnoreCase("componentPropertyCacheRepoGroups") && (entity.indexOf("SiteNode") > -1 || entity.indexOf("ContentVersion") > -1 || entity.indexOf("AccessRight") > 0 || entity.indexOf("SystemUser") > 0 || entity.indexOf("Role") > 0 || entity.indexOf("Group") > 0)) { clear = true; if (entity.indexOf("SiteNode") > -1 || entity.indexOf("ContentVersion") > -1) selectiveCacheUpdate = true; } if (cacheName.equalsIgnoreCase("componentPropertyVersionIdCacheRepoGroups") && (entity.indexOf("SiteNode") > -1 || entity.indexOf("ContentVersion") > -1 || entity.indexOf("AccessRight") > 0 || entity.indexOf("SystemUser") > 0 || entity.indexOf("Role") > 0 || entity.indexOf("Group") > 0)) { clear = true; if (entity.indexOf("SiteNode") > -1 || entity.indexOf("ContentVersion") > -1) selectiveCacheUpdate = true; } if (cacheName.equalsIgnoreCase("pageComponentsCache") && (entity.indexOf("ContentVersion") > -1 || entity.indexOf("AccessRight") > 0 || entity.indexOf("SystemUser") > 0 || entity.indexOf("Role") > 0 || entity.indexOf("Group") > 0)) { clear = true; //selectiveCacheUpdate = true; } if (cacheName.equalsIgnoreCase("includeCache")) { clear = true; } if (cacheName.equalsIgnoreCase("authorizationCache") && (entity.indexOf("AccessRight") > 0 || entity.indexOf("SystemUser") > 0 || entity.indexOf("Role") > 0 || entity.indexOf("Group") > 0 || entity.indexOf("Intercept") > 0)) { clear = true; } if (cacheName.equalsIgnoreCase("personalAuthorizationCache") && (entity.indexOf("AccessRight") > 0 || entity.indexOf("SystemUser") > 0 || entity.indexOf("Role") > 0 || entity.indexOf("Group") > 0 || entity.indexOf("Intercept") > 0)) { clear = true; } if (cacheName.equalsIgnoreCase("componentPaletteDivCache") && (entity.indexOf("AccessRight") > 0 || entity.indexOf("SystemUser") > 0 || entity.indexOf("Role") > 0 || entity.indexOf("Group") > 0)) { clear = true; } if (cacheName.equalsIgnoreCase("userCache") && (entity.indexOf("AccessRight") > 0 || entity.indexOf("SystemUser") > 0 || entity.indexOf("Role") > 0 || entity.indexOf("Group") > 0)) { clear = true; } if (cacheName.equalsIgnoreCase("principalCache") && (entity.indexOf("SystemUser") > 0 || entity.indexOf("Role") > 0 || entity.indexOf("Group") > 0)) { clear = true; } if ((cacheName.equalsIgnoreCase("assetUrlCache") || cacheName.equalsIgnoreCase("assetUrlCacheWithGroups") || cacheName.equalsIgnoreCase("assetThumbnailUrlCache")) && (entity.indexOf("DigitalAsset") > 0 || entity.indexOf("ContentVersion") > 0 || entity.indexOf("AccessRight") > 0 || entity.indexOf("SystemUser") > 0 || entity.indexOf("Role") > 0 || entity.indexOf("Group") > 0)) { clear = true; if (cacheName.equalsIgnoreCase("assetUrlCacheWithGroups") && (entity.indexOf("ContentVersion") > -1 || entity.indexOf("DigitalAsset") > -1)) selectiveCacheUpdate = true; } if (cacheName.equalsIgnoreCase("digitalAssetCache") && (entity.indexOf("DigitalAsset") > 0 || entity.indexOf("ContentVersion") > 0)) { clear = true; } if (cacheName.equalsIgnoreCase("sortedChildContentsCache") && (entity.indexOf("Content") > 0 || entity.indexOf("ContentVersion") > 0 || entity.indexOf("AccessRight") > 0 || entity.indexOf("SystemUser") > 0 || entity.indexOf("Role") > 0 || entity.indexOf("Group") > 0)) { clear = true; } if (cacheName.equalsIgnoreCase("childContentCache") && (entity.indexOf("Content") > 0 || entity.indexOf("ContentVersion") > 0 || entity.indexOf("AccessRight") > 0 || entity.indexOf("SystemUser") > 0 || entity.indexOf("Role") > 0 || entity.indexOf("Group") > 0)) { clear = true; selectiveCacheUpdate = true; } if (cacheName.equalsIgnoreCase("matchingContentsCache") && (entity.indexOf("Content") > 0 || entity.indexOf("ContentVersion") > 0 || entity.indexOf("AccessRight") > 0 || entity.indexOf("SystemUser") > 0 || entity.indexOf("Role") > 0 || entity.indexOf("Group") > 0)) { clear = true; selectiveCacheUpdate = true; } if (cacheName.equalsIgnoreCase("workflowCache") && entity.indexOf("WorkflowDefinition") > 0) { clear = true; } if (cacheName.equalsIgnoreCase("rootSiteNodeCache") && entity.indexOf("SiteNode") > 0) { if (CmsPropertyHandler.getOperatingMode().equals("0")) clear = true; } if ((cacheName.equalsIgnoreCase("siteNodeCache") || cacheName.equalsIgnoreCase("siteNodeVOCache")) && entity.indexOf("SiteNode") > 0) { clear = true; selectiveCacheUpdate = true; } if (cacheName.equalsIgnoreCase("contentCache") && entity.indexOf("Content") > 0) { clear = true; selectiveCacheUpdate = true; } if (cacheName.equalsIgnoreCase("rootContentCache") && entity.indexOf("Content") > 0) { clear = true; } if (cacheName.equalsIgnoreCase("componentContentsCache") && entity.indexOf("Content") > 0) { clear = true; } if (cacheName.equalsIgnoreCase("childSiteNodesCache") && (entity.indexOf("SiteNode") > 0 || entity.indexOf("ContentVersion") > 0)) { clear = true; selectiveCacheUpdate = true; } if (cacheName.equalsIgnoreCase("childPagesCache") && (entity.indexOf("SiteNode") > 0 || entity.indexOf("Content") > 0)) { clear = true; selectiveCacheUpdate = true; } if (cacheName.equalsIgnoreCase("siteNodeCacheWithLatestVersion") && entity.indexOf("SiteNode") > 0) { clear = true; selectiveCacheUpdate = true; } if (cacheName.equalsIgnoreCase("propertySetCache") && entity.indexOf("SiteNode") > 0) { clear = true; } if (cacheName.equalsIgnoreCase("groupVOListCache") && entity.indexOf("Group") > 0) { clear = true; } if (cacheName.equalsIgnoreCase("roleListCache") && entity.indexOf("Role") > 0) { clear = true; } if (cacheName.equalsIgnoreCase("groupPropertiesCache") && (entity.indexOf("Group") > 0 || entity.indexOf("PropertiesCategory") > 0)) { clear = true; } if (cacheName.equalsIgnoreCase("propertiesCategoryCache") && (entity.indexOf("Group") > 0 || entity.indexOf("Role") > 0 || entity.indexOf("User") > 0 || entity.indexOf("PropertiesCategory") > 0)) { clear = true; } if (cacheName.equalsIgnoreCase("rolePropertiesCache") && entity.indexOf("Role") > 0) { clear = true; } if (cacheName.equalsIgnoreCase("principalPropertyValueCache") && (entity.indexOf("Group") > 0 || entity.indexOf("Role") > 0 || entity.indexOf("User") > 0)) { clear = true; } if (cacheName.equalsIgnoreCase("relatedCategoriesCache") && (entity.indexOf("Group") > 0 || entity.indexOf("Role") > 0 || entity.indexOf("User") > 0 || entity.indexOf("PropertiesCategory") > 0)) { clear = true; } if (cacheName.equalsIgnoreCase("categoryCache") && entity.indexOf("Category") > 0) { clear = true; } if (cacheName.equalsIgnoreCase("contentCategoryCache") && entity.indexOf("ContentVersion") > 0) { clear = true; selectiveCacheUpdate = true; } if (cacheName.equalsIgnoreCase("redirectCache") && entity.indexOf("Redirect") > 0) { clear = true; } if (cacheName.equalsIgnoreCase("interceptorsCache") && entity.indexOf("Intercept") > 0) { clear = true; } if (cacheName.equalsIgnoreCase("interceptionPointCache") && entity.indexOf("Intercept") > 0) { clear = true; } if (cacheName.equalsIgnoreCase("siteNodeLanguageCache") && (entity.indexOf("Repository") > 0 || entity.indexOf("Language") > 0 || entity.indexOf("SiteNode") > 0)) { clear = true; selectiveCacheUpdate = true; } if (cacheName.equalsIgnoreCase("contentTypeDefinitionCache") && entity.indexOf("ContentTypeDefinition") > 0) { clear = true; } if (cacheName.equalsIgnoreCase("ServerNodeProperties")) { clear = true; } if (!cacheName.equalsIgnoreCase("serverNodePropertiesCache") && entity.equalsIgnoreCase("ServerNodeProperties")) { clear = true; } if (!cacheName.equalsIgnoreCase("encodedStringsCache") && entity.equalsIgnoreCase("ServerNodeProperties")) { clear = true; } if (logger.isInfoEnabled()) logger.info("clear:" + clear); if (clear) { if (logger.isInfoEnabled()) logger.info("clearing:" + e.getKey()); Object object = e.getValue(); String sentContentId = null; String sentParentContentId = null; String sentSiteNodeId = null; String sentParentSiteNodeId = null; String sentRepositoryId = null; String sentContentTypeDefinitionId = null; String sentContentIsProtected = null; if (extraInformation != null) { sentContentId = extraInformation.get("contentId"); sentParentContentId = extraInformation.get("parentContentId"); sentSiteNodeId = extraInformation.get("siteNodeId"); sentParentSiteNodeId = extraInformation.get("parentSiteNodeId"); sentRepositoryId = extraInformation.get("repositoryId"); sentContentTypeDefinitionId = extraInformation.get("contentTypeDefinitionId"); sentContentIsProtected = extraInformation.get("contentIsProtected"); } //System.out.println("sentContentId:" + sentContentId); //System.out.println("sentParentContentId:" + sentParentContentId); //System.out.println("sentSiteNodeId:" + sentSiteNodeId); //System.out.println("sentParentSiteNodeId:" + sentParentSiteNodeId); //System.out.println("sentRepositoryId:" + sentRepositoryId); if (object instanceof Map) { Map cacheInstance = (Map) e.getValue(); synchronized (cacheInstance) { if (cacheName.equals("componentContentsCache")) { try { if (entity.indexOf("ContentVersion") > 0) { String contentId = sentContentId; String contentTypeDefinitionId = sentContentTypeDefinitionId; if (contentId == null || contentId.equals("")) { try { contentId = "" + ContentVersionController .getContentVersionController() .getContentIdForContentVersion(new Integer(entityId)); } catch (Exception e2) { logger.info("Error loading content with id " + entityId + ":" + e2.getMessage()); } } if (contentId != null) { //ContentVO contentVO = ContentController.getContentController().getContentVOWithId(contentId); if (contentTypeDefinitionId != null && !contentTypeDefinitionId.equals("")) { ContentTypeDefinitionVO ctdVO = ContentTypeDefinitionController .getController().getContentTypeDefinitionVOWithId( new Integer(contentTypeDefinitionId)); if (ctdVO.getName().equals("HTMLTemplate") || ctdVO.getName().equals("PagePartTemplate")) { ComponentController.getController() .reIndexComponentContentsDelayed( new Integer(contentId)); } } else logger.info("No content type for " + contentId); } } else logger.info("skipping clearing components as it seems stupid"); } catch (Exception e2) { logger.warn("Error clearing componentContentsCache:" + e2.getMessage(), e2); } } else if (!(cacheName.equals("userAccessCache") && cacheInstance.size() < 100)) { logger.info( "clearing ordinary map:" + e.getKey() + " (" + cacheInstance.size() + ")"); cacheInstance.clear(); } else logger.info("skipping clearing this as it seems stupid"); } } else { GeneralCacheAdministrator cacheInstance = (GeneralCacheAdministrator) e.getValue(); synchronized (cacheInstance) //Back { //t.printElapsedTime("START:" + entity + ":" + entityId); //ADD logic to flush correct on sitenode and sitenodeversion /* if(selectiveCacheUpdate && entity.indexOf("SiteNode") > 0) { cacheInstance.flushAll(); eventListeners.remove(cacheName + "_cacheEntryEventListener"); eventListeners.remove(cacheName + "_cacheMapAccessEventListener"); logger.info("clearing:" + e.getKey()); } */ //System.out.println("entity:" + entity); if (entity.indexOf("pageCache") == 0) { if (entity.indexOf("pageCache:") == 0) { String groupQualifyer = entity.substring("pageCache:".length()); logger.info( "CacheController: This is a application pageCache-clear request... specific:" + groupQualifyer); logger.info("clearing " + e.getKey() + " : " + groupQualifyer); PageCacheHelper.getInstance().notify("" + groupQualifyer); /* if(cacheName.equals("pageCacheExtra")) { clearFileCacheForGroup(cacheInstance, "" + groupQualifyer); } else if(cacheName.equals("pageCache")) { cacheInstance.flushGroup("" + groupQualifyer); } */ } else { PageCacheHelper.getInstance().notify("selectiveCacheUpdateNonApplicable"); /* logger.error("clearing " + e.getKey() + " selectiveCacheUpdateNonApplicable"); if(cacheName.equals("pageCacheExtra")) { clearFileCacheForGroup(cacheInstance, "selectiveCacheUpdateNonApplicable"); } else if(cacheName.equals("pageCache")) { cacheInstance.flushGroup("selectiveCacheUpdateNonApplicable"); } */ } } if (selectiveCacheUpdate && entity.indexOf("Repository") > 0 && useSelectivePageCacheUpdate) { if (cacheName.equals("pageCacheExtra")) { PageCacheHelper.getInstance().notify("repository_" + entityId); PageCacheHelper.getInstance().notify("selectiveCacheUpdateNonApplicable"); } else { cacheInstance.flushGroup("repository_" + entityId); cacheInstance.flushGroup("selectiveCacheUpdateNonApplicable"); } /* logger.info("clearing " + e.getKey() + " with group " + "repository_" + entityId); if(cacheName.equals("pageCacheExtra")) { clearFileCacheForGroup(cacheInstance, "repository_" + entityId); clearFileCacheForGroup(cacheInstance, "selectiveCacheUpdateNonApplicable"); } else { cacheInstance.flushGroup("repository_" + entityId); cacheInstance.flushGroup("selectiveCacheUpdateNonApplicable"); } */ } else if (selectiveCacheUpdate && entity.indexOf("SiteNodeVersion") > 0) { //Thread.dumpStack(); //How to solve this good if (CmsPropertyHandler.getOperatingMode().equalsIgnoreCase("0")) { if (cacheName.equals("pageCacheExtra")) { PageCacheHelper.getInstance().notify("siteNodeVersion_" + entityId); PageCacheHelper.getInstance().notify("selectiveCacheUpdateNonApplicable"); //clearFileCacheForGroup(cacheInstance, "siteNodeVersion_" + entityId); //clearFileCacheForGroup(cacheInstance, "selectiveCacheUpdateNonApplicable"); } else { cacheInstance.flushGroup("siteNodeVersion_" + entityId); cacheInstance.flushGroup("selectiveCacheUpdateNonApplicable"); } logger.info("clearing " + e.getKey() + " with group " + "siteNodeVersion_" + entityId); try { logger.info("BeforesiteNodeVersionVO..."); String siteNodeId = sentSiteNodeId; String repositoryId = sentRepositoryId; String parentSiteNodeId = sentParentSiteNodeId; //System.out.println("siteNodeId:" + siteNodeId); //System.out.println("repositoryId:" + repositoryId); //System.out.println("parentSiteNodeId:" + parentSiteNodeId); if (siteNodeId == null || siteNodeId.equals("")) { try { SiteNodeVersionVO snvVO = SiteNodeVersionController.getController() .getSiteNodeVersionVOWithId(new Integer(entityId)); siteNodeId = "" + snvVO.getSiteNodeId(); if (repositoryId == null) { SiteNodeVO snVO = SiteNodeController.getController() .getSiteNodeVOWithId(snvVO.getSiteNodeId()); repositoryId = "" + snVO.getRepositoryId(); parentSiteNodeId = "" + snVO.getParentSiteNodeId(); } } catch (Exception e2) { logger.info("Error getting siteNodeVersion " + entityId); } } if (siteNodeId != null) { logger.info("Before flushGroup2..."); if (cacheName.equals("pageCacheExtra")) { PageCacheHelper.getInstance().notify("siteNode_" + siteNodeId); //clearFileCacheForGroup(cacheInstance, "siteNode_" + siteNodeId); } else { cacheInstance.flushGroup("siteNode_" + siteNodeId); cacheInstance.flushGroup("" + siteNodeId); } if (siteNodeId != null && (cacheName.equals("childSiteNodesCache") || cacheName.equals("childPagesCache") || cacheName.equals("siteNodeCache") || cacheName.equals("componentPropertyCacheRepoGroups") || cacheName .equals("componentPropertyVersionIdCacheRepoGroups"))) { if (cacheName.equals("componentPropertyCacheRepoGroups") || cacheName.equals( "componentPropertyVersionIdCacheRepoGroups")) { cacheInstance.flushGroup("" + repositoryId); logger.info( "Clearing componentPropertyCacheRepoGroups for repo:" + repositoryId); } if (parentSiteNodeId != null) { cacheInstance.flushGroup("siteNode_" + parentSiteNodeId); cacheInstance.flushGroup("" + parentSiteNodeId); cacheInstance.flushEntry("" + parentSiteNodeId); logger.info("Clearing for:" + parentSiteNodeId); } } logger.info("After flushGroup2..."); } } catch (Exception se) { logger.warn("Missing siteNode version: " + se.getMessage(), se); } } else { try { if ((cacheName.equals("childSiteNodesCache") || cacheName.equals("childPagesCache") || cacheName.equals("siteNodeCache") || cacheName.equals("componentPropertyCacheRepoGroups") || cacheName.equals("componentPropertyVersionIdCacheRepoGroups"))) { SiteNodeVersionVO snvVO = SiteNodeVersionController.getController() .getSiteNodeVersionVOWithId(new Integer(entityId)); SiteNodeVO snVO = SiteNodeController.getController() .getSiteNodeVOWithId(snvVO.getSiteNodeId()); Integer repositoryId = snVO.getRepositoryId(); Integer parentSiteNodeId = snVO.getParentSiteNodeId(); if (cacheName.equals("componentPropertyCacheRepoGroups") || cacheName .equals("componentPropertyVersionIdCacheRepoGroups")) { cacheInstance.flushGroup("" + repositoryId); logger.info("Clearing componentPropertyCacheRepoGroups for repo:" + repositoryId); } if (parentSiteNodeId != null) { cacheInstance.flushGroup("siteNode_" + parentSiteNodeId); cacheInstance.flushGroup("" + parentSiteNodeId); cacheInstance.flushEntry("" + parentSiteNodeId); logger.info("Clearing for:" + parentSiteNodeId); } } } catch (Exception e2) { logger.error( "Problem clearing cache for site node version:" + e2.getMessage(), e2); } } } else if (selectiveCacheUpdate && (entity.indexOf("SiteNode") > 0 && entity.indexOf("SiteNodeTypeDefinition") == -1) && useSelectivePageCacheUpdate) { //System.out.println("Entity: " + entity); logger.info("Flushing " + "" + entityId); logger.info("Flushing " + "siteNode_" + entityId); logger.info("Flushing " + "selectiveCacheUpdateNonApplicable"); if (cacheName.equals("pageCacheExtra")) { PageCacheHelper.getInstance().notify("siteNode_" + entityId); PageCacheHelper.getInstance().notify("selectiveCacheUpdateNonApplicable"); //clearFileCacheForGroup(cacheInstance, "siteNode_" + entityId); //clearFileCacheForGroup(cacheInstance, "selectiveCacheUpdateNonApplicable"); } else { cacheInstance.flushEntry("" + entityId); cacheInstance.flushGroup("" + entityId); cacheInstance.flushGroup("siteNode_" + entityId); cacheInstance.flushGroup("selectiveCacheUpdateNonApplicable"); if (cacheName.equals("childSiteNodesCache") || cacheName.equals("childPagesCache") || cacheName.equals("siteNodeCache") || cacheName.equals("componentPropertyCacheRepoGroups") || cacheName.equals("componentPropertyVersionIdCacheRepoGroups")) { logger.info("Flushing parent also"); String repositoryId = sentRepositoryId; String parentSiteNodeId = sentParentSiteNodeId; try { if (repositoryId == null) { SiteNodeVO snVO = SiteNodeController.getController() .getSiteNodeVOWithId(new Integer(entityId)); if (snVO != null) { repositoryId = "" + snVO.getRepositoryId(); parentSiteNodeId = "" + snVO.getParentSiteNodeId(); } } if (cacheName.equals("componentPropertyCacheRepoGroups") || cacheName .equals("componentPropertyVersionIdCacheRepoGroups")) { cacheInstance.flushGroup("" + repositoryId); logger.info("Clearing componentPropertyCacheRepoGroups for repo:" + repositoryId); } if (parentSiteNodeId != null && !parentSiteNodeId.equals("")) { logger.info("Flushing " + "" + entityId); logger.info("Flushing " + "siteNode_" + entityId); cacheInstance.flushGroup("siteNode_" + parentSiteNodeId); cacheInstance.flushGroup("" + parentSiteNodeId); cacheInstance.flushEntry("" + parentSiteNodeId); logger.info("Clearing for:" + parentSiteNodeId); } } catch (SystemException se) { logger.warn("Missing siteNode: " + se.getMessage(), se); } } } logger.info("clearing " + e.getKey() + " with group " + "siteNode_" + entityId); } else if (selectiveCacheUpdate && entity.indexOf("ContentVersion") > 0 && useSelectivePageCacheUpdate) { //t.printElapsedTime("CV start...."); logger.info("ContentVersion entity was sent: " + entity + ":" + entityId + " and cacheName:" + cacheName); logger.info("Getting eventListeners..."); //Object cacheEntryEventListener = eventListeners.get(e.getKey() + "_cacheEntryEventListener"); //Object cacheMapAccessEventListener = eventListeners.get(e.getKey() + "_cacheMapAccessEventListener"); //System.out.println("entity:" + entity); //System.out.println("Before flushGroup:" +cacheName); logger.info("Before flushGroup..."); if (cacheName.equals("pageCacheExtra")) { //clearFileCacheForGroup(cacheInstance, "contentVersion_" + entityId); //clearFileCacheForGroup(cacheInstance, "selectiveCacheUpdateNonApplicable"); } else if (cacheName.equals("pageCache")) { logger.info("Skipping clearing pageCache for version"); //cacheInstance.flushGroup("contentVersion_" + entityId); //cacheInstance.flushGroup("selectiveCacheUpdateNonApplicable"); } else if (cacheName.equals("componentPropertyCacheRepoGroups") || cacheName.equals("componentPropertyVersionIdCacheRepoGroups")) { Timer t2 = new Timer(); try { String repositoryId = sentRepositoryId; if (repositoryId == null) { String contentId = sentContentId; if (contentId == null) contentId = "" + ContentVersionController .getContentVersionController() .getContentIdForContentVersion(new Integer(entityId)); ContentVO contentVO = ContentController.getContentController() .getContentVOWithId(new Integer(contentId)); repositoryId = "" + contentVO.getRepositoryId(); } cacheInstance.flushGroup("" + repositoryId); //t2.printElapsedTime("3"); if (cacheName.equals("componentPropertyVersionIdCacheRepoGroups")) cacheInstance.flushGroup("selectiveCacheUpdateNonApplicable"); //t2.printElapsedTime("4"); logger.info("Clearing componentPropertyCacheRepoGroups for repo:" + repositoryId); } catch (Exception e2) { logger.info("Error loading content with id " + entityId + ":" + e2.getMessage()); } //t.printElapsedTime("componentPropertyCacheRepoGroups"); } else if (cacheName.equals("assetUrlCacheWithGroups")) { try { String contentId = sentContentId; if (contentId == null || contentId.equals("")) contentId = "" + ContentVersionController.getContentVersionController() .getContentIdForContentVersion(new Integer(entityId)); cacheInstance.flushGroup("content_" + contentId); logger.info("Clearing assetUrlCacheWithGroups for content:" + "content_" + contentId); } catch (Exception e2) { logger.warn( "Flushing assetUrlCacheWithGroups as it was a missing entity - was probably a delete"); cacheInstance.flushAll(); } //t.printElapsedTime("assetUrlCacheWithGroups"); } else if (cacheName.equals("childPagesCache") || cacheName.equals("childSiteNodesCache")) { //System.out.println("childPagesCache:" + entity + "=" + entityId); //t.printElapsedTime("childPagesCache start"); try { String contentId = sentContentId; //System.out.println("sentContentId:" + sentContentId); if (contentId == null || contentId.equals("")) contentId = "" + ContentVersionController.getContentVersionController() .getContentIdForContentVersion(new Integer(entityId)); //System.out.println("contentId:" + contentId); //t.printElapsedTime("childPagesCache A"); ContentTypeDefinitionVO metaInfoContentTypeDefinitionVO = ContentTypeDefinitionController .getController().getContentTypeDefinitionVOWithName("Meta info"); //t.printElapsedTime("childPagesCache B"); ContentVO contentVO = ContentController.getContentController() .getContentVOWithId(new Integer(contentId)); //t.printElapsedTime("childPagesCache C"); if (metaInfoContentTypeDefinitionVO.getId() .equals(contentVO.getContentTypeDefinitionId())) { try { SiteNodeVO siteNodeVO = SiteNodeController.getController() .getSiteNodeVOWithMetaInfoContentId(new Integer(contentId)); //t.printElapsedTime("childPagesCache getSiteNodeVOWithMetaInfoContentId"); if (siteNodeVO != null) { cacheInstance.flushGroup("siteNode_" + siteNodeVO.getId()); cacheInstance.flushGroup( "siteNode_" + siteNodeVO.getParentSiteNodeId()); } } catch (Exception e2) { logger.error( "Did not find a sitenode with this meta info:" + contentId); } cacheInstance.flushGroup("content_" + contentId); cacheInstance.flushGroup("contentVersion_" + entityId); cacheInstance.flushGroup("selectiveCacheUpdateNonApplicable"); //t.printElapsedTime("childPagesCache flush done..."); logger.info("Clearing childPagesCache for content:" + "content_" + contentId); } } catch (Exception e2) { logger.warn( "Flushing childPagesCache as it was a missing entity - was probably a delete"); cacheInstance.flushAll(); } //t.printElapsedTime("childPagesCache"); } else if (cacheName.equals("matchingContentsCache")) { try { String contentId = sentContentId; if (contentId == null || contentId.equals("")) contentId = "" + ContentVersionController.getContentVersionController() .getContentIdForContentVersion(new Integer(entityId)); try { ContentVO contentVO = ContentController.getContentController() .getContentVOWithId(new Integer(contentId)); String contentTypeDefinitionId = "" + contentVO.getContentTypeDefinitionId(); cacheInstance.flushGroup( "selectiveCacheUpdateNonApplicable_contentTypeDefinitionId_" + contentTypeDefinitionId); cacheInstance.putInCache("recacheMark_" + contentTypeDefinitionId, "" + System.currentTimeMillis()); cacheInstance.putInCache("recacheMark", "" + System.currentTimeMillis()); } catch (Exception e2) { logger.warn( "Flushing all as it was a missing entity - was probably a delete:" + e2.getMessage()); cacheInstance.flushAll(); } cacheInstance.flushGroup("content_" + contentId); logger.info("Clearing assetUrlCacheWithGroups for content:" + "content_" + contentId); } catch (Exception e2) { logger.warn( "Flushing all as it was a missing entity - was probably a delete:" + e2.getMessage()); cacheInstance.flushAll(); } //t.printElapsedTime("matchingContentsCache"); } else { //t.printElapsedTime("Before"); cacheInstance.flushGroup("contentVersion_" + entityId); //if(!cacheName.equals("contentCache") && !cacheName.equals("contentVersionCache") && !cacheName.equals("contentAttributeCache") && !cacheName.equals("contentVersionIdCache") && !cacheName.equals("contentCategoryCache") && !cacheName.equals("metaInfoContentAttributeCache")) cacheInstance.flushGroup("selectiveCacheUpdateNonApplicable"); logger.info( "clearing " + e.getKey() + " with selectiveCacheUpdateNonApplicable"); //t.printElapsedTime("clearing " + e.getKey() + " with selectiveCacheUpdateNonApplicable"); } logger.info( "clearing " + e.getKey() + " with group " + "contentVersion_" + entityId); //String[] changedAttributes = new String[]{"Title","NavigationTitle"}; try { //t.printElapsedTime("Cache 3.4"); logger.info("Before contentVersionVO..."); //System.out.println("cacheName:" + cacheName); //System.out.println("entity:" + entity); //System.out.println("entityId:" + entityId); String contentIdString = sentContentId; String contentTypeDefinitionId = sentContentTypeDefinitionId; String contentIsProtected = sentContentIsProtected; if (contentIdString == null || contentIdString.equals("")) { try { contentIdString = "" + ContentVersionController.getContentVersionController() .getContentIdForContentVersion(new Integer(entityId)); } catch (Exception e2) { logger.info("Error loading content with id " + entityId + ":" + e2.getMessage()); } } Integer contentId = null; if (contentIdString != null) contentId = new Integer(contentIdString); if (contentTypeDefinitionId == null && contentIsProtected == null && contentId != null) { ContentVO contentVO = ContentController.getContentController() .getContentVOWithId(new Integer(contentId)); contentTypeDefinitionId = "" + contentVO.getContentTypeDefinitionId(); contentIsProtected = "" + contentVO.getIsProtected().intValue(); } //t.printElapsedTime("Cache 3.5"); //RequestAnalyser.getRequestAnalyser().registerComponentStatistics("Cache 3.5", t.getElapsedTime()); if (contentId != null) { List<String> changes = Collections.EMPTY_LIST; //System.out.println("extraInformation:" + extraInformation); String changedAttributes = extraInformation == null ? null : extraInformation.get("changedAttributeNames"); if (changedAttributes != null && changedAttributes.length() > 0) changes = new ArrayList<String>( Arrays.asList(StringUtils.split(changedAttributes, ","))); //ContentVO contentVO = ContentController.getContentController().getContentVOWithId(new Integer(contentId)); logger.info("Before flushGroup2..."); if (cacheName.equals("pageCacheExtra")) { if (contentIsProtected.equals("" + ContentVO.YES.intValue())) { List<InterceptionPointVO> interceptionPointVOList = InterceptionPointController .getController().getInterceptionPointVOList("Content"); for (InterceptionPointVO interceptionPointVO : interceptionPointVOList) { if (interceptionPointVO.getName().endsWith(".Read")) { String acKey = "" + interceptionPointVO.getId() + "_" + entityId; CacheController.clearUserAccessCache(acKey); } } } PageCacheHelper.getInstance().notify( "selectiveCacheUpdateNonApplicable_contentTypeDefinitionId_" + contentTypeDefinitionId); PageCacheHelper.getInstance().notify("content_" + contentId); //clearFileCacheForGroup(cacheInstance, "selectiveCacheUpdateNonApplicable_contentTypeDefinitionId_" + contentVO.getContentTypeDefinitionId()); if ((changes == null || changes.size() == 0) && CmsPropertyHandler.getOperatingMode().equals("3")) { ContentVersionVO oldContentVersionVO = ContentVersionController .getContentVersionController() .getContentVersionVOWithId(new Integer(entityId)); ContentVersionVO newContentVersionVO = ContentVersionController .getContentVersionController() .getLatestActiveContentVersionVO(contentId, oldContentVersionVO.getLanguageId(), new Integer(CmsPropertyHandler.getOperatingMode())); if (newContentVersionVO != null && oldContentVersionVO != null && newContentVersionVO.getId() .equals(oldContentVersionVO.getId())) { oldContentVersionVO = null; //System.out.println("SHIT - same version allready - must find other"); List<SmallestContentVersionVO> contentVersionVOList = ContentVersionController .getContentVersionController() .getSmallestContentVersionVOList( new Integer(contentId)); for (SmallestContentVersionVO cvVO : contentVersionVOList) { if (!cvVO.getId().equals(newContentVersionVO.getId()) && cvVO.getStateId() .equals(new Integer(CmsPropertyHandler .getOperatingMode())) && cvVO.getLanguageId() .equals(newContentVersionVO.getLanguageId()) && cvVO.getIsActive() && (oldContentVersionVO == null || oldContentVersionVO.getId() < cvVO .getId())) { oldContentVersionVO = ContentVersionController .getContentVersionController() .getContentVersionVOWithId(cvVO.getId()); } } } //System.out.println("Now we should have current and previous version:" + newContentVersionVO + " / " + oldContentVersionVO); if (newContentVersionVO != null && oldContentVersionVO != null) changes = ContentVersionController.getContentVersionController() .getChangedAttributeNames(newContentVersionVO, oldContentVersionVO); } //System.out.println("changes:" + changes); for (String changedAttributeName : changes) { if (changedAttributeName.indexOf("ComponentStructure") > -1) { //Map allreadyFlushedEntries.... Set<String> groupEntries = (Set<String>) cacheInstance .getCache().cacheMap .getGroup("content_" + contentId + "_ComponentStructureDependency"); //System.out.println("groupEntries:" + groupEntries); if (groupEntries != null) { System.out.println("groupEntries:" + groupEntries.size()); outer: for (String key : groupEntries) { //System.out.println("key 1:" + key); try { //String[] usedEntities = (String[])cacheInstance.getFromCache(key + "_entities"); byte[] usedEntitiesByteArray = (byte[]) cacheInstance .getFromCache(key + "_entitiesAsByte"); String usedEntitiesString = compressionHelper .decompress(usedEntitiesByteArray); //t.printElapsedTime("Decompress to " + usedEntitiesString.length() + " took"); String[] usedEntities = StringUtils .split(usedEntitiesString, "|"); //t.printElapsedTime("Split to usedEntities " + usedEntities.length + " took"); ContentVersionVO newContentVersionVO = ContentVersionController .getContentVersionController() .getContentVersionVOWithId( new Integer(entityId)); String newComponentStructure = ContentVersionController .getContentVersionController() .getAttributeValue(newContentVersionVO, "ComponentStructure", false); for (String usedEntity : usedEntities) { //System.out.println("usedEntity:" + usedEntity); if (usedEntity.startsWith("content_" + contentId + "_ComponentStructure(")) { //System.out.println("Match - now lets parse: " + usedEntity); String arguments = usedEntity.substring( usedEntity.indexOf("(") + 1, usedEntity.indexOf(")")); Integer oldComponentPropertyHash = new Integer( usedEntity.substring( usedEntity.indexOf("=") + 1)); String[] args = arguments.split(","); Integer componentId = new Integer(args[0]); String propertyName = args[1]; Integer siteNodeId = new Integer(args[2]); Integer languageId = new Integer(args[3]); //System.out.println("componentId:" + componentId); //System.out.println("propertyName:" + propertyName); //System.out.println("siteNodeId:" + siteNodeId); //System.out.println("languageId:" + languageId); int newComponentPropertyHash = getPropertyAsStringHashCode( newComponentStructure, componentId, propertyName, siteNodeId, languageId); //System.out.println("oldComponentPropertyHash:" + oldComponentPropertyHash); //System.out.println("newComponentPropertyHash:" + newComponentPropertyHash); if (oldComponentPropertyHash .intValue() != newComponentPropertyHash) { //System.out.println("Yes - clearing - must have changed something important:" + usedEntity); PageCacheHelper.getInstance() .notify(usedEntity); //clearFileCacheForGroup(cacheInstance, usedEntity); } else { //System.out.println("Flushing content_" + currentPageMetaInfoContentId + "_ComponentStructure just to catch page itself"); //cacheInstance.flushGroup("content_" + currentPageMetaInfoContentId + "_ComponentStructure"); //System.out.println("Flushing content_" + contentId + "_ComponentStructure just to catch page itself"); //cacheInstance.flushGroup("content_" + contentId + "_ComponentStructure"); } } else if (usedEntity.startsWith("content_" + contentId + "_ComponentStructure:")) { //System.out.println("Match - now lets parse component order etc: " + usedEntity); String xPath = usedEntity.substring( usedEntity.indexOf(":") + 1, usedEntity.lastIndexOf("=")); Integer oldComponentPropertyHash = new Integer( usedEntity.substring( usedEntity.lastIndexOf("=") + 1)); //System.out.println("xPath:" + xPath); int newComponentPropertyHash = getComponentsAsStringHashCode( newComponentStructure, xPath); //System.out.println("oldComponentPropertyHash:" + oldComponentPropertyHash); //System.out.println("newComponentPropertyHash:" + newComponentPropertyHash); if (oldComponentPropertyHash .intValue() != newComponentPropertyHash) { //System.out.println("Yes - clearing - must have changed order or added/subtracted components:" + usedEntity); PageCacheHelper.getInstance() .notify(usedEntity); //clearFileCacheForGroup(cacheInstance, usedEntity); } } } } catch (Exception ex) { //logger.error("Got error trying to update cache:" + ex.getMessage()); logger.warn("Got error trying to update cache:" + ex.getMessage(), ex); //clearFileCacheForGroup(cacheInstance, "content_" + contentId + "_" + changedAttributeName); //cacheInstance.flushGroup("content_" + contentId + "_" + changedAttributeName); //logger.warn("Cleared pageCache for " + "content_" + contentId + "_" + changedAttributeName); break outer; } } } } else { PageCacheHelper.getInstance().notify( "content_" + contentId + "_" + changedAttributeName); //clearFileCacheForGroup(cacheInstance, "content_" + contentId + "_" + changedAttributeName); //System.out.println("Cleared for " + "content_" + contentId + "_" + changedAttributeName); } } //t.printElapsedTime("Handled page cache extra"); RequestAnalyser.getRequestAnalyser().registerComponentStatistics( "Handled page cache extra", t.getElapsedTime()); //clearFileCacheForGroup(cacheInstance, "content_" + contentId); } else if (cacheName.equals("pageCache")) { //t.printElapsedTime("Page cache start"); RequestAnalyser.getRequestAnalyser().registerComponentStatistics( "Page cache start", t.getElapsedTime()); logger.info("Flushing pageCache for content type def"); String contentTypeDefKey = "selectiveCacheUpdateNonApplicable_contentTypeDefinitionId_" + contentTypeDefinitionId; cacheInstance.flushGroup(contentTypeDefKey); //cacheInstance.flushGroup("selectiveCacheUpdateNonApplicable_contentTypeDefinitionId_" + contentVO.getContentTypeDefinitionId()); //cacheInstance.flushGroup("content_" + contentVO.getId()); //System.out.println("Flushing:" + getPooledString(1, contentVO.getId())); cacheInstance.flushGroup(getPooledString(1, new Integer(contentId))); PageCacheHelper.getInstance().notify("content_" + contentId); ContentVersionVO oldContentVersionVO = null; ContentVersionVO newContentVersionVO = null; String debug = ""; if ((changes == null || changes.size() == 0) && CmsPropertyHandler.getOperatingMode().equals("3")) { debug += "entityId:" + entityId + "\n"; debug += "contentId:" + contentId + "\n"; oldContentVersionVO = ContentVersionController .getContentVersionController() .getContentVersionVOWithId(new Integer(entityId)); debug += "oldContentVersionVO:" + oldContentVersionVO.getId() + ":" + oldContentVersionVO.getLanguageId() + "\n"; debug += "oldContentVersionVO:" + CmsPropertyHandler.getOperatingMode() + "\n"; newContentVersionVO = ContentVersionController .getContentVersionController() .getLatestActiveContentVersionVO(contentId, oldContentVersionVO.getLanguageId(), new Integer(CmsPropertyHandler.getOperatingMode())); debug += "newContentVersionVO:" + newContentVersionVO + "\n"; if (newContentVersionVO != null && oldContentVersionVO != null && newContentVersionVO.getId() .equals(oldContentVersionVO.getId())) { debug += "newContentVersionVO:" + newContentVersionVO.getId() + "\n"; oldContentVersionVO = null; debug += "SHIT - same version allready - must find other"; List<SmallestContentVersionVO> contentVersionVOList = ContentVersionController .getContentVersionController() .getSmallestContentVersionVOList( new Integer(contentId)); for (SmallestContentVersionVO cvVO : contentVersionVOList) { if (!cvVO.getId().equals(newContentVersionVO.getId()) && cvVO.getStateId() .equals(new Integer(CmsPropertyHandler .getOperatingMode())) && cvVO.getLanguageId() .equals(newContentVersionVO.getLanguageId()) && cvVO.getIsActive() && (oldContentVersionVO == null || oldContentVersionVO.getId() < cvVO .getId())) { oldContentVersionVO = ContentVersionController .getContentVersionController() .getContentVersionVOWithId(cvVO.getId()); } } debug += "oldContentVersionVO:" + (oldContentVersionVO == null ? "null" : oldContentVersionVO.getId()) + "\n"; } //System.out.println("Now we should have current and previous version:" + newContentVersionVO + " / " + oldContentVersionVO); if (newContentVersionVO != null && oldContentVersionVO != null) changes = ContentVersionController.getContentVersionController() .getChangedAttributeNames(newContentVersionVO, oldContentVersionVO); } //t.printElapsedTime("Changes analyzed"); //RequestAnalyser.getRequestAnalyser().registerComponentStatistics("Changes analyzed", t.getElapsedTime()); if ((changes == null || changes.size() == 0) && CmsPropertyHandler.getOperatingMode().equals("3")) { if (oldContentVersionVO == null || newContentVersionVO == null) { //Hur kan det bli detta???? logger.warn("Fishy 1: " + oldContentVersionVO + ":" + newContentVersionVO + " in " + CmsPropertyHandler.getContextRootPath()); logger.warn("DEBUG: " + debug); } else { logger.warn( "Fishy 2: No changes found between content versions " + newContentVersionVO.getId() + " and " + oldContentVersionVO.getId() + " in " + CmsPropertyHandler.getContextRootPath()); logger.warn("DEBUG: " + debug); logger.warn("Fishy: newContentVersionVO: " + newContentVersionVO.getVersionValue()); logger.warn("Fishy: newContentVersionVO: " + oldContentVersionVO.getVersionValue()); } logger.warn( "Just to make sure pages are updated we pretend all attributes changed until we find the bug"); changes = ContentVersionController.getContentVersionController() .getAttributeNames(newContentVersionVO); } //System.out.println("changes:" + changes); for (String changedAttributeName : changes) { logger.warn("changedAttributeName: " + changedAttributeName); if (changedAttributeName.indexOf("ComponentStructure") > -1 && cacheName.equals("pageCache")) { //Map allreadyFlushedEntries.... //It's something wrong here.. GeneralCacheAdministrator pageCacheExtraInstance = (GeneralCacheAdministrator) caches .get("pageCacheExtra"); String cacheGroupKey = "content_" + contentId + "_ComponentStructureDependency"; //Set<String> groupEntries = (Set<String>)cacheInstance.getCache().cacheMap.getGroup("content_" + contentId + "_ComponentStructureDependency"); Set<String> groupEntries = (Set<String>) cacheInstance .getCache().cacheMap.getGroup( getPooledString(cacheGroupKey.hashCode())); //System.out.println("groupEntries:" + groupEntries); if (groupEntries != null) { outer: for (String key : groupEntries) { logger.info("key 2:" + key); try { //String[] usedEntities = (String[])pageCacheExtraInstance.getFromCache(key + "_entities"); byte[] usedEntitiesByteArray = (byte[]) pageCacheExtraInstance .getFromCache(key + "_entitiesAsByte"); String usedEntitiesString = compressionHelper .decompress(usedEntitiesByteArray); //t.printElapsedTime("Decompress to " + usedEntitiesString.length() + " took"); String[] usedEntities = StringUtils .split(usedEntitiesString, ","); //t.printElapsedTime("Split to usedEntities " + usedEntities.length + " took"); ContentVersionVO newestContentVersionVO = ContentVersionController .getContentVersionController() .getContentVersionVOWithId( new Integer(entityId)); String newComponentStructure = ContentVersionController .getContentVersionController() .getAttributeValue(newestContentVersionVO, "ComponentStructure", false); for (String usedEntity : usedEntities) { //System.out.println("usedEntity:" + usedEntity); if (usedEntity.startsWith("content_" + contentId + "_ComponentStructure(")) { //System.out.println("Match - now lets parse: " + usedEntity); String arguments = usedEntity.substring( usedEntity.indexOf("(") + 1, usedEntity.indexOf(")")); Integer oldComponentPropertyHash = new Integer( usedEntity.substring( usedEntity.indexOf("=") + 1)); String[] args = arguments.split(","); Integer componentId = new Integer(args[0]); String propertyName = args[1]; Integer siteNodeId = new Integer(args[2]); Integer languageId = new Integer(args[3]); //System.out.println("componentId:" + componentId); //System.out.println("propertyName:" + propertyName); //System.out.println("siteNodeId:" + siteNodeId); //System.out.println("languageId:" + languageId); int newComponentPropertyHash = getPropertyAsStringHashCode( newComponentStructure, componentId, propertyName, siteNodeId, languageId); //System.out.println("oldComponentPropertyHash:" + oldComponentPropertyHash); //System.out.println("newComponentPropertyHash:" + newComponentPropertyHash); if (oldComponentPropertyHash .intValue() != newComponentPropertyHash) { //System.out.println("Yes - clearing - must have changed something important:" + usedEntity); //cacheInstance.flushGroup(usedEntity); cacheInstance .flushGroup(getPooledString( usedEntity.hashCode())); //clearFileCacheForGroup(cacheInstance, usedEntity); } else { //System.out.println("Flushing content_" + currentPageMetaInfoContentId + "_ComponentStructure just to catch page itself"); //cacheInstance.flushGroup("content_" + currentPageMetaInfoContentId + "_ComponentStructure"); //System.out.println("Flushing content_" + contentId + "_ComponentStructure just to catch page itself"); String componentStructureKey = "content_" + contentId + "_ComponentStructure"; //cacheInstance.flushGroup(componentStructureKey); cacheInstance .flushGroup(getPooledString( componentStructureKey .hashCode())); cacheInstance .flushGroup(getPooledString( usedEntity.hashCode())); } } else if (usedEntity.startsWith("content_" + contentId + "_ComponentStructure:")) { //System.out.println("Match - now lets parse component order etc: " + usedEntity); String xPath = usedEntity.substring( usedEntity.indexOf(":") + 1, usedEntity.lastIndexOf("=")); Integer oldComponentPropertyHash = new Integer( usedEntity.substring( usedEntity.lastIndexOf("=") + 1)); //System.out.println("xPath:" + xPath); int newComponentPropertyHash = getComponentsAsStringHashCode( newComponentStructure, xPath); //System.out.println("oldComponentPropertyHash:" + oldComponentPropertyHash); //System.out.println("newComponentPropertyHash:" + newComponentPropertyHash); if (oldComponentPropertyHash .intValue() != newComponentPropertyHash) { //System.out.println("Yes - clearing - must have changed order or added/subtracted components:" + usedEntity); cacheInstance .flushGroup(getPooledString( usedEntity.hashCode())); //cacheInstance.flushGroup(usedEntity); } } } } catch (Exception ex) { //logger.error("Got error trying to update cache:" + ex.getMessage()); logger.warn("Got error trying to update cache:" + ex.getMessage(), ex); try { String attributeKey = "content_" + contentId + "_" + changedAttributeName; //cacheInstance.flushGroup(attributeKey); cacheInstance.flushGroup(getPooledString( attributeKey.hashCode())); logger.warn("Cleared pageCache for " + getPooledString( attributeKey.hashCode())); } catch (Exception ex2) { logger.error("Got error trying to flushGroup 2:" + ex2.getMessage()); cacheInstance.flushAll(); } break outer; } t.printElapsedTime("Handled group entries"); //RequestAnalyser.getRequestAnalyser().registerComponentStatistics("Handled group entries", t.getElapsedTime()); } } } else { String attributeKey = "content_" + contentId + "_" + changedAttributeName; //cacheInstance.flushGroup("content_" + contentId + "_" + changedAttributeName); cacheInstance .flushGroup(getPooledString(attributeKey.hashCode())); logger.info("Cleared pageCache for " + "content_" + contentId + "_" + changedAttributeName); } } //cacheInstance.flushGroup("content_" + contentId); } else { String contentTypeDefKey = "selectiveCacheUpdateNonApplicable_contentTypeDefinitionId_" + contentTypeDefinitionId; //cacheInstance.flushGroup(contentTypeDefKey); cacheInstance.flushGroup(contentTypeDefKey); //System.out.println("Cleared for " + "content_" + contentId + " on cache " + cacheName); String contentKey = "content_" + contentId; cacheInstance.flushGroup(contentKey); //cacheInstance.flushGroup(contentKey); } //t.printElapsedTime("Handled page cache"); //RequestAnalyser.getRequestAnalyser().registerComponentStatistics("Handled page cache", t.getElapsedTime()); logger.info("After flushGroup2..."); } //} if (cacheName.equals("contentVersionCache")) { new AssetUpdatingThread(entityId).start(); } } catch (SystemException se) { se.printStackTrace(); logger.info("Missing content version: " + se.getMessage()); } catch (Exception ex) { ex.printStackTrace(); } } else if (selectiveCacheUpdate && (entity.indexOf("Content") > 0 && entity.indexOf("ContentTypeDefinition") == -1 && entity.indexOf("ContentCategory") == -1) && useSelectivePageCacheUpdate) { logger.info("Content entity was sent: " + entity + ":" + entityId); //System.out.println("Content entity was called and needs to be fixed:" + entity); //String[] changedAttributes = new String[]{"Title","NavigationTitle"}; /* ContentVO contentVO = null; if(isObjectCachedInCastor(SmallContentImpl.class, new Integer(entityId))) contentVO = ContentController.getContentController().getContentVOWithId(new Integer(entityId)); */ String repositoryId = sentRepositoryId; String contentTypeDefinitionId = sentContentTypeDefinitionId; String contentIsProtected = sentContentIsProtected; if (repositoryId == null || contentTypeDefinitionId == null || contentIsProtected == null) { //System.out.println("repositoryId:" + repositoryId); //System.out.println("contentTypeDefinitionId:" + contentTypeDefinitionId); //System.out.println("contentIsProtected:" + contentIsProtected); try { ContentVO contentVO = ContentController.getContentController() .getContentVOWithId(new Integer(entityId)); //t.printElapsedTime("clearCaches cv u1 contentVO", 10); repositoryId = "" + contentVO.getRepositoryId(); contentTypeDefinitionId = "" + contentVO.getContentTypeDefinitionId(); contentIsProtected = "" + contentVO.getIsProtected(); } catch (Exception e2) { logger.info("Error loading content with id " + entityId + ":" + e2.getMessage()); } } if (cacheName.equals("componentPropertyCacheRepoGroups") || cacheName.equals("componentPropertyVersionIdCacheRepoGroups")) { cacheInstance.flushGroup("" + repositoryId); logger.info( "Clearing componentPropertyCacheRepoGroups for repo:" + repositoryId); } if (cacheName.equals("pageCacheExtra")) { //clearFileCacheForGroup(cacheInstance, "content_" + entityId); PageCacheHelper.getInstance().notify("content_" + entityId); try { //cacheInstance.flushGroup("selectiveCacheUpdateNonApplicable_contentTypeDefinitionId_" + contentVO.getContentTypeDefinitionId()); if (contentTypeDefinitionId != null) { //clearFileCacheForGroup(cacheInstance, "selectiveCacheUpdateNonApplicable_contentTypeDefinitionId_" + contentVO.getContentTypeDefinitionId()); PageCacheHelper.getInstance().notify( "selectiveCacheUpdateNonApplicable_contentTypeDefinitionId_" + contentTypeDefinitionId); } } catch (Exception e2) { logger.warn("Could not find content type to clear pages based on: " + e2.getMessage(), e2); } } else if (cacheName.equals("pageCache")) { logger.info("Flushing page cache for {" + entityId + "} and {content_" + entityId + "}"); String entityKey = "" + entityId; String contentEntityKey = "content_" + entityId; //cacheInstance.flushGroup("" + entityId); //cacheInstance.flushGroup(contentEntityKey); cacheInstance.flushGroup(entityKey); cacheInstance.flushGroup(contentEntityKey); try { if (contentTypeDefinitionId != null) { String contentTypeCacheKey = "selectiveCacheUpdateNonApplicable_contentTypeDefinitionId_" + contentTypeDefinitionId; //cacheInstance.flushGroup(contentTypeCacheKey); cacheInstance.flushGroup(contentTypeCacheKey); } } catch (Exception e2) { logger.warn("Could not find content type to clear pages based on: " + e2.getMessage(), e2); } } else { cacheInstance.flushGroup("" + entityId); cacheInstance.flushGroup("content_" + entityId); logger.info( "clearing " + e.getKey() + " with selectiveCacheUpdateNonApplicable"); cacheInstance.flushGroup("selectiveCacheUpdateNonApplicable"); if (contentTypeDefinitionId != null) { cacheInstance.flushGroup( "selectiveCacheUpdateNonApplicable_contentTypeDefinitionId_" + contentTypeDefinitionId); } } if (contentTypeDefinitionId != null) { //System.out.println("****************************************************************"); if (contentIsProtected.equals("" + ContentVO.YES.intValue())) { List<InterceptionPointVO> interceptionPointVOList = InterceptionPointController .getController().getInterceptionPointVOList("Content"); for (InterceptionPointVO interceptionPointVO : interceptionPointVOList) { if (interceptionPointVO.getName().endsWith(".Read")) { String acKey = "" + interceptionPointVO.getId() + "_" + entityId; //System.out.println("Clearing access rights for:" + acKey); CacheController.clearUserAccessCache(acKey); } } } } //System.out.println("************************END************************************"); logger.info("clearing " + e.getKey() + " with group " + "content_" + entityId); } else if (selectiveCacheUpdate && entity.indexOf("DigitalAsset") > -1) { logger.info("Asset entity was sent: " + entity + ":" + entityId); Integer contentId = assetContentIdMapping.get(new Integer(entityId)); if (contentId == null) { logger.info("Checking fot cv for asset:" + entityId); List<SmallestContentVersionVO> contentVersions = DigitalAssetController .getContentVersionVOListConnectedToAssetWithId(new Integer(entityId)); if (contentVersions != null) { for (SmallestContentVersionVO contentVersionVO : contentVersions) { contentId = contentVersionVO.getContentId(); assetContentIdMapping.put(new Integer(entityId), contentId); break; } } } else logger.info("Using read asset"); //Integer contentId = null; ContentVO contentVO = null; try { contentVO = ContentController.getContentController() .getContentVOWithId(contentId); if (cacheName.equals("componentPropertyCacheRepoGroups") || cacheName.equals("componentPropertyVersionIdCacheRepoGroups")) { cacheInstance.flushGroup("" + contentVO.getRepositoryId()); logger.info("Clearing componentPropertyCacheRepoGroups for repo:" + contentVO.getRepositoryId()); } } catch (Exception e2) { logger.info( "Error loading content with id " + contentId + ":" + e2.getMessage()); } //ContentVO contentVO = ContentController.getContentController().getContentVOWithId(new Integer(entityId)); if (cacheName.equals("pageCacheExtra")) { //clearFileCacheForGroup(cacheInstance, "content_" + entityId); PageCacheHelper.getInstance().notify("content_" + contentId); try { //cacheInstance.flushGroup("selectiveCacheUpdateNonApplicable_contentTypeDefinitionId_" + contentVO.getContentTypeDefinitionId()); if (contentVO != null) { //clearFileCacheForGroup(cacheInstance, "selectiveCacheUpdateNonApplicable_contentTypeDefinitionId_" + contentVO.getContentTypeDefinitionId()); PageCacheHelper.getInstance().notify( "selectiveCacheUpdateNonApplicable_contentTypeDefinitionId_" + contentVO.getContentTypeDefinitionId()); } } catch (Exception e2) { logger.warn("Could not find content type to clear pages based on: " + e2.getMessage(), e2); } } else if (cacheName.equals("pageCache")) { logger.info("Flushing page cache for {" + contentId + "} and {content_" + contentId + "}"); String entityKey = "" + contentId; String contentEntityKey = "content_" + contentId; //cacheInstance.flushGroup("" + entityId); //cacheInstance.flushGroup(contentEntityKey); cacheInstance.flushGroup(entityKey); cacheInstance.flushGroup(contentEntityKey); try { if (contentVO != null) { String contentTypeCacheKey = "selectiveCacheUpdateNonApplicable_contentTypeDefinitionId_" + contentVO.getContentTypeDefinitionId(); //cacheInstance.flushGroup(contentTypeCacheKey); cacheInstance.flushGroup(contentTypeCacheKey); } } catch (Exception e2) { logger.warn("Could not find content type to clear pages based on: " + e2.getMessage(), e2); } } else { cacheInstance.flushGroup("" + contentId); cacheInstance.flushGroup("content_" + contentId); logger.info( "clearing " + e.getKey() + " with selectiveCacheUpdateNonApplicable"); cacheInstance.flushGroup("selectiveCacheUpdateNonApplicable"); } logger.info("clearing " + e.getKey() + " with group " + "content_" + entityId); } else if (selectiveCacheUpdate && entity.indexOf("Publication") > 0 && useSelectivePageCacheUpdate && (operatingMode != null && operatingMode.equalsIgnoreCase("3")) && CmsPropertyHandler.getLivePublicationThreadClass().equalsIgnoreCase( "org.infoglue.deliver.util.SelectiveLivePublicationThread")) { logger.info("Now we will ease out the publication..."); /* List publicationDetailVOList = PublicationController.getController().getPublicationDetailVOList(new Integer(entityId)); Iterator publicationDetailVOListIterator = publicationDetailVOList.iterator(); while(publicationDetailVOListIterator.hasNext()) { PublicationDetailVO publicationDetailVO = (PublicationDetailVO)publicationDetailVOListIterator.next(); logger.info("publicationDetailVO.getEntityClass():" + publicationDetailVO.getEntityClass()); logger.info("publicationDetailVO.getEntityId():" + publicationDetailVO.getEntityId()); if(Class.forName(publicationDetailVO.getEntityClass()).getName().equals(ContentVersion.class.getName())) { logger.error("We clear all caches having references to contentVersion: " + publicationDetailVO.getEntityId()); Integer contentId = ContentVersionController.getContentVersionController().getContentIdForContentVersion(publicationDetailVO.getEntityId()); cacheInstance.flushGroup("content_" + contentId); cacheInstance.flushGroup(CacheController.getPooledString(2, publicationDetailVO.getEntityId().toString())); cacheInstance.flushGroup("selectiveCacheUpdateNonApplicable"); logger.info("clearing " + e.getKey() + " with group " + "content_" + contentId); logger.info("clearing " + e.getKey() + " with group " + "content_" + contentId); } else if(Class.forName(publicationDetailVO.getEntityClass()).getName().equals(SiteNodeVersion.class.getName())) { Integer siteNodeId = SiteNodeVersionController.getController().getSiteNodeVersionVOWithId(publicationDetailVO.getEntityId()).getSiteNodeId(); CacheController.clearCaches(publicationDetailVO.getEntityClass(), publicationDetailVO.getEntityId().toString(), null); } } */ } else if (entity .equals("org.infoglue.cms.entities.management.impl.simple.AccessRightImpl")) { //System.out.println("This was an access right update - do we handle it:" + cacheName); if (!CmsPropertyHandler.getOperatingMode().equalsIgnoreCase("3")) { try { AccessRightVO acVO = AccessRightController.getController() .getAccessRightVOWithId(new Integer(entityId)); InterceptionPointVO icpVO = InterceptionPointController.getController() .getInterceptionPointVOWithId(acVO.getInterceptionPointId()); //System.out.println("icpVO:" + icpVO.getName()); if (icpVO.getName().indexOf("Content.") > -1) { //System.out.println("Was a content access... let's clear caches for that content."); String idAsString = acVO.getParameters(); if (idAsString != null && !idAsString.equals("")) clearCaches( "org.infoglue.cms.entities.content.impl.simple.ContentImpl", idAsString, null, cachesToSkip, forceClear); } else if (icpVO.getName().indexOf("ContentVersion.") > -1) { //System.out.println("Was a contentversion access... let's clear caches for that content."); String idAsString = acVO.getParameters(); if (idAsString != null && !idAsString.equals("")) clearCaches( "org.infoglue.cms.entities.content.impl.simple.ContentVersionImpl", idAsString, null, cachesToSkip, forceClear); } else if (icpVO.getName().indexOf("SiteNode.") > -1) { //System.out.println("Was a sitenode access... let's clear caches for that content."); String idAsString = acVO.getParameters(); if (idAsString != null && !idAsString.equals("")) clearCaches( "org.infoglue.cms.entities.structure.impl.simple.SiteNodeImpl", idAsString, null, cachesToSkip, forceClear); } else if (icpVO.getName().indexOf("SiteNodeVersion.") > -1) { //System.out.println("Was a sitenode version access... let's clear caches for that content."); String idAsString = acVO.getParameters(); if (idAsString != null && !idAsString.equals("")) clearCaches( "org.infoglue.cms.entities.structure.impl.simple.SiteNodeVersionImpl", idAsString, null, cachesToSkip, forceClear); } else { //System.out.println("****************************"); //System.out.println("* WHAT TO DO WITH IN CACHECONTROLLER: " + icpVO.getName() + " *"); //System.out.println("****************************"); } } catch (Exception e2) { logger.error("Error handling access right update: " + e2.getMessage(), e2); } } else logger.info("Skipping it as this is live mode.."); } else if (selectiveCacheUpdate && (entity.indexOf("ServiceBinding") > 0 || entity.indexOf("Qualifyer") > 0)) { logger.info( "Ignoring this kind of notification... never used anymore:" + cacheName); } else if (entity.indexOf("AccessRightImpl") > -1) { logger.info("Ignoring handling of entity:" + entity); } else if (cacheName.equalsIgnoreCase("matchingContentsCache") && entity.indexOf("MediumContentCategoryImpl") > 0) { try { String contentId = sentContentId; //if(contentId == null || contentId.equals("")) ContentCategoryVO contentCategoryVO = ContentCategoryController.getController() .findById(new Integer(entityId)); ContentVersionVO contentVersionVO = ContentVersionController .getContentVersionController() .getContentVersionVOWithId(contentCategoryVO.getContentVersionId()); ContentVO contentVO = ContentController.getContentController() .getContentVOWithId(contentVersionVO.getContentId()); String contentTypeDefinitionId = "" + contentVO.getContentTypeDefinitionId(); cacheInstance .flushGroup("selectiveCacheUpdateNonApplicable_contentTypeDefinitionId_" + contentTypeDefinitionId); cacheInstance.putInCache("recacheMark_" + contentTypeDefinitionId, "" + System.currentTimeMillis()); cacheInstance.putInCache("recacheMark", "" + System.currentTimeMillis()); } catch (Exception e2) { cacheInstance.putInCache("recacheAllMark", "" + System.currentTimeMillis()); logger.warn("Flushing all as it was a missing entity - was probably a delete: " + e2.getMessage()); cacheInstance.flushAll(); } } else if (entity.indexOf("MediumContentCategoryImpl") > 0) { logger.info("Special handling - no handling"); } else if (cacheName.equalsIgnoreCase("componentEditorVersionIdCache") && entity.indexOf("ContentVersionImpl") > 0) { logger.info("Special handling componentEditorVersionIdCache"); String contentId = sentContentId; if (contentId == null || contentId.equals("")) contentId = "" + ContentVersionController.getContentVersionController() .getContentIdForContentVersion(new Integer(entityId)); cacheInstance.flushGroup("content_" + contentId); cacheInstance.flushGroup("contentVersion_" + entityId); } else if ((cacheName.equalsIgnoreCase("componentEditorCache") || cacheName.equalsIgnoreCase("componentPropertyCache") || cacheName.equalsIgnoreCase("componentPropertyVersionIdCache") || cacheName.equalsIgnoreCase("pageComponentsCache")) && entity.indexOf("ContentVersionImpl") > 0) { try { logger.info("Special handling componentEditorVersionIdCache"); String contentId = sentContentId; if (contentId == null || contentId.equals("")) contentId = "" + ContentVersionController.getContentVersionController() .getContentIdForContentVersion(new Integer(entityId)); ContentVO contentVO = ContentController.getContentController() .getContentVOWithId(new Integer(contentId)); if (contentVO != null && contentVO.getContentTypeDefinitionId() != null) { ContentTypeDefinitionVO ctdVO = ContentTypeDefinitionController .getController().getContentTypeDefinitionVOWithId( contentVO.getContentTypeDefinitionId()); if (ctdVO.getName().equals("Meta info")) cacheInstance.flushAll(); else { if (cacheName.equalsIgnoreCase("componentEditorCache") && (ctdVO.getName().equals("HTMLTemplate") || ctdVO.getName().equals("PagePartTemplate"))) { cacheInstance.flushAll(); //CacheController.clearCache("componentEditorCache"); } else logger.info("No need to clear page stuff"); } } } catch (Exception e2) { if (e2.getCause() instanceof ObjectNotFoundException || (e2.getCause() != null && e2.getCause().getCause() instanceof ObjectNotFoundException)) logger.info("Error clearing caches - object was probably deleted."); else logger.warn("Error handling mixed caches: " + e2.getMessage(), e2); } } else { if (entity.indexOf("EventImpl") == -1 && entity.indexOf(".Publication") == -1) { if (logger.isInfoEnabled()) logger.info("Clearing all on: " + cacheName + ":" + entity); cacheInstance.flushAll(); if (logger.isInfoEnabled()) logger.info("clearing:" + e.getKey()); } } } //BACK } long elapsedTime = t.getElapsedTime(); if (elapsedTime > 20) logger.info("Clear cache end took " + e.getKey() + ": " + elapsedTime); logger.info("Cleared cache:" + e.getKey()); if (!selectiveCacheUpdate) i.remove(); } else { logger.info("Did not clear " + e.getKey()); } } //} if (!useSelectivePageCacheUpdate || entity.indexOf("AccessRight") > -1) { logger.info("Clearing all pageCaches"); CacheController.clearFileCaches("pageCache"); } } t.printElapsedTime("clearCaches stop", 100); /* logger.info("clearCaches stop"); long time = t.getElapsedTime(); if(time > 3000) logger.warn("clearCaches took long time:" + time); */ }
From source file:snpviewer.SnpViewer.java
public void writeSavedRegionsToFile() { if (savedRegions.size() < 1) { Dialogs.showErrorDialog(null, "No Saved Regions exist to write!", "No Saved Regions", "SnpViewer"); return;//from w w w . ja v a2 s . com } final int flanks = 10; FileChooser fileChooser = new FileChooser(); FileChooser.ExtensionFilter extFilter = new FileChooser.ExtensionFilter("Excel (*.xlsx)", "*.xlsx"); fileChooser.getExtensionFilters().add(extFilter); fileChooser.setTitle("Write regions to Excel file (.xlsx)..."); File rFile = fileChooser.showSaveDialog(mainWindow); if (rFile == null) { return; } else if (!rFile.getName().endsWith(".xlsx")) { rFile = new File(rFile.getAbsolutePath() + ".xlsx"); } final File regionFile = rFile; final Task<Boolean> writeTask = new Task() { @Override protected Boolean call() throws Exception { try { updateProgress(-1, -1); BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(regionFile)); Workbook wb = new XSSFWorkbook(); //first create a summary sheet of all regions Sheet sheet = wb.createSheet(); Row row = null; int rowNo = 0; int sheetNo = 0; wb.setSheetName(sheetNo++, "Summary"); row = sheet.createRow(rowNo++); String header[] = { "Coordinates", "rsIDs", "Size (Mb)" }; for (int col = 0; col < header.length; col++) { Cell cell = row.createCell(col); cell.setCellValue(header[col]); } for (int i = 0; i < savedRegions.size(); i++) { row = sheet.createRow(rowNo++); int col = 0; Cell cell = row.createCell(col++); cell.setCellValue("chr" + savedRegions.get(i).getCoordinateString()); cell = row.createCell(col++); cell.setCellValue(savedRegions.get(i).getIdLine()); cell = row.createCell(col++); double mB = (double) savedRegions.get(i).getLength() / 1000000; cell.setCellValue(mB); } ArrayList<SnpFile> bothFiles = new ArrayList<>(); bothFiles.addAll(affFiles); bothFiles.addAll(unFiles); String prevChrom = new String(); double prog = 0; double total = savedRegions.size() * bothFiles.size() * 2; updateProgress(prog, total); int regCounter = 0; for (RegionSummary reg : savedRegions) { updateMessage("Writing region " + ++regCounter + " of " + savedRegions.size()); //create a sheet for each chromosome if (!reg.getChromosome().equalsIgnoreCase(prevChrom)) { if (!prevChrom.isEmpty()) { CellRangeAddress[] regions = { new CellRangeAddress(0, rowNo, 2, 2 + bothFiles.size()) }; SheetConditionalFormatting sheetCF = sheet.getSheetConditionalFormatting(); ConditionalFormattingRule rule1 = sheetCF .createConditionalFormattingRule(ComparisonOperator.EQUAL, "\"AA\""); PatternFormatting fill1 = rule1.createPatternFormatting(); fill1.setFillBackgroundColor(IndexedColors.LIGHT_GREEN.index); fill1.setFillPattern(PatternFormatting.SOLID_FOREGROUND); ConditionalFormattingRule rule2 = sheetCF .createConditionalFormattingRule(ComparisonOperator.EQUAL, "\"BB\""); PatternFormatting fill2 = rule2.createPatternFormatting(); fill2.setFillBackgroundColor(IndexedColors.PALE_BLUE.index); fill2.setFillPattern(PatternFormatting.SOLID_FOREGROUND); ConditionalFormattingRule rule3 = sheetCF .createConditionalFormattingRule(ComparisonOperator.EQUAL, "\"AB\""); PatternFormatting fill3 = rule3.createPatternFormatting(); fill3.setFillBackgroundColor(IndexedColors.ROSE.index); fill3.setFillPattern(PatternFormatting.SOLID_FOREGROUND); sheetCF.addConditionalFormatting(regions, rule3, rule2); sheetCF.addConditionalFormatting(regions, rule1); } rowNo = 0; sheet = wb.createSheet(); wb.setSheetName(sheetNo++, reg.getChromosome()); prevChrom = reg.getChromosome(); } else {//pad regions with an empty line rowNo++; } TreeMap<Integer, HashMap<String, String>> coordMap = new TreeMap(); /*coordmap - key is position, key of hashmap * is input filename and value call */ HashMap<Integer, String> coordToId = new HashMap<>(); //coordinate to rs ID try { for (SnpFile f : bothFiles) { updateProgress(prog++, total); if (isCancelled()) { return false; } List<SnpFile.SnpLine> lines = f.getSnpsInRegion(reg.getChromosome(), reg.getStartPos(), reg.getEndPos(), flanks); for (SnpFile.SnpLine snpLine : lines) { if (isCancelled()) { return false; } Integer coord = snpLine.getPosition(); if (!coordMap.containsKey(coord)) { coordMap.put(coord, new HashMap<String, String>()); } String filename = f.inputFile.getName(); String rsId = snpLine.getId(); String call = snpLine.getCall(); coordMap.get(coord).put(filename, call); coordToId.put(coord, rsId); } } row = sheet.createRow(rowNo++); Cell cell = row.createCell(0); cell.setCellValue(reg.getCoordinateString()); row = sheet.createRow(rowNo++); cell = row.createCell(0); cell.setCellValue(reg.getIdLine()); int col = 0; row = sheet.createRow(rowNo++); cell = row.createCell(col++); cell.setCellValue("Position"); cell = row.createCell(col++); cell.setCellValue("rsID"); for (SnpFile f : bothFiles) { updateProgress(prog++, total); cell = row.createCell(col++); if (f.getSampleName() != null && !f.getSampleName().isEmpty()) { cell.setCellValue(f.getSampleName()); } else { cell.setCellValue(f.inputFile.getName()); } } for (Entry current : coordMap.entrySet()) { if (isCancelled()) { return false; } col = 0; Integer coord = (Integer) current.getKey(); row = sheet.createRow(rowNo++); cell = row.createCell(col++); cell.setCellValue(coord); cell = row.createCell(col++); cell.setCellValue(coordToId.get(coord)); HashMap<String, String> fileToCall = (HashMap<String, String>) current.getValue(); for (SnpFile f : bothFiles) { cell = row.createCell(col++); if (fileToCall.containsKey(f.inputFile.getName())) { cell.setCellValue(fileToCall.get(f.inputFile.getName())); } else { cell.setCellValue("-"); } } } } catch (Exception ex) { return false; } } CellRangeAddress[] regions = { new CellRangeAddress(0, rowNo, 2, 2 + bothFiles.size()) }; SheetConditionalFormatting sheetCF = sheet.getSheetConditionalFormatting(); ConditionalFormattingRule rule1 = sheetCF .createConditionalFormattingRule(ComparisonOperator.EQUAL, "\"AA\""); PatternFormatting fill1 = rule1.createPatternFormatting(); fill1.setFillBackgroundColor(IndexedColors.LIGHT_GREEN.index); fill1.setFillPattern(PatternFormatting.SOLID_FOREGROUND); ConditionalFormattingRule rule2 = sheetCF .createConditionalFormattingRule(ComparisonOperator.EQUAL, "\"BB\""); PatternFormatting fill2 = rule2.createPatternFormatting(); fill2.setFillBackgroundColor(IndexedColors.PALE_BLUE.index); fill2.setFillPattern(PatternFormatting.SOLID_FOREGROUND); ConditionalFormattingRule rule3 = sheetCF .createConditionalFormattingRule(ComparisonOperator.EQUAL, "\"AB\""); PatternFormatting fill3 = rule3.createPatternFormatting(); fill3.setFillBackgroundColor(IndexedColors.ROSE.index); fill3.setFillPattern(PatternFormatting.SOLID_FOREGROUND); sheetCF.addConditionalFormatting(regions, rule3, rule2); sheetCF.addConditionalFormatting(regions, rule1); wb.write(out); updateProgress(total, total); out.close(); } catch (IOException | NumberFormatException ex) { ex.printStackTrace(); return false; } return true; } };//end of task setProgressMode(true); progressBar.progressProperty().bind(writeTask.progressProperty()); progressMessage.textProperty().bind(writeTask.messageProperty()); writeTask.setOnSucceeded(new EventHandler<WorkerStateEvent>() { @Override public void handle(WorkerStateEvent e) { if (e.getSource().getValue() == true) { Dialogs.showInformationDialog(null, "Saved regions written " + "to file " + "(" + regionFile.getName() + ")successfully", "Regions Written", "SNP Viewer"); } else { Dialogs.showErrorDialog(null, "Region write failed.", "Write Failed", "SNP Viewer"); } setProgressMode(false); progressBar.progressProperty().unbind(); progressBar.progressProperty().set(0); progressMessage.textProperty().unbind(); progressMessage.setText(""); progressTitle.setText(""); } }); writeTask.setOnFailed(new EventHandler<WorkerStateEvent>() { @Override public void handle(WorkerStateEvent e) { setProgressMode(false); progressBar.progressProperty().unbind(); progressBar.progressProperty().set(0); progressMessage.textProperty().unbind(); progressMessage.setText(""); progressTitle.setText("Region write failed!"); Dialogs.showErrorDialog(null, "Error writing region to file\n", "Region write error", "SNP Viewer", e.getSource().getException()); } }); writeTask.setOnCancelled(new EventHandler<WorkerStateEvent>() { @Override public void handle(WorkerStateEvent e) { progressMessage.setText("Region write cancelled"); progressTitle.setText("Cancelled"); setProgressMode(false); progressBar.progressProperty().unbind(); progressBar.progressProperty().set(0); Dialogs.showErrorDialog(null, "Error writing region to file\n", "Region write error", "SNP Viewer"); } }); cancelButton.setOnAction(new EventHandler<ActionEvent>() { @Override public void handle(ActionEvent actionEvent) { writeTask.cancel(); } }); progressTitle.setText("Writing regions to .xlsx file"); new Thread(writeTask).start(); }
From source file:org.opendatakit.services.database.utlities.ODKDatabaseImplUtils.java
/** * If the caller specified a complex json value for a structured type, flush * the value through to the individual columns. * * @param orderedColumns//from w w w . j av a 2 s.c o m * @param values */ private void cleanUpValuesMap(OrderedColumns orderedColumns, Map<String, Object> values) { TreeMap<String, String> toBeResolved = new TreeMap<String, String>(); for (String key : values.keySet()) { if (DataTableColumns.CONFLICT_TYPE.equals(key)) { continue; } else if (DataTableColumns.FILTER_TYPE.equals(key)) { continue; } else if (DataTableColumns.FILTER_VALUE.equals(key)) { continue; } else if (DataTableColumns.FORM_ID.equals(key)) { continue; } else if (DataTableColumns.ID.equals(key)) { continue; } else if (DataTableColumns.LOCALE.equals(key)) { continue; } else if (DataTableColumns.ROW_ETAG.equals(key)) { continue; } else if (DataTableColumns.SAVEPOINT_CREATOR.equals(key)) { continue; } else if (DataTableColumns.SAVEPOINT_TIMESTAMP.equals(key)) { continue; } else if (DataTableColumns.SAVEPOINT_TYPE.equals(key)) { continue; } else if (DataTableColumns.SYNC_STATE.equals(key)) { continue; } else if (DataTableColumns._ID.equals(key)) { continue; } // OK it is one of the data columns ColumnDefinition cp = orderedColumns.find(key); if (!cp.isUnitOfRetention()) { toBeResolved.put(key, (String) values.get(key)); } } // remove these non-retained values from the values set... for (String key : toBeResolved.keySet()) { values.remove(key); } while (!toBeResolved.isEmpty()) { TreeMap<String, String> moreToResolve = new TreeMap<String, String>(); for (TreeMap.Entry<String, String> entry : toBeResolved.entrySet()) { String key = entry.getKey(); String json = entry.getValue(); if (json == null) { // don't need to do anything // since the value is null continue; } ColumnDefinition cp = orderedColumns.find(key); try { TypeReference<Map<String, Object>> reference = new TypeReference<Map<String, Object>>() { }; Map<String, Object> struct = ODKFileUtils.mapper.readValue(json, reference); for (ColumnDefinition child : cp.getChildren()) { String subkey = child.getElementKey(); ColumnDefinition subcp = orderedColumns.find(subkey); if (subcp.isUnitOfRetention()) { ElementType subtype = subcp.getType(); ElementDataType type = subtype.getDataType(); if (type == ElementDataType.integer) { values.put(subkey, (Integer) struct.get(subcp.getElementName())); } else if (type == ElementDataType.number) { values.put(subkey, (Double) struct.get(subcp.getElementName())); } else if (type == ElementDataType.bool) { values.put(subkey, ((Boolean) struct.get(subcp.getElementName())) ? 1 : 0); } else { values.put(subkey, (String) struct.get(subcp.getElementName())); } } else { // this must be a javascript structure... re-JSON it and save (for // next round). moreToResolve.put(subkey, ODKFileUtils.mapper.writeValueAsString(struct.get(subcp.getElementName()))); } } } catch (JsonParseException e) { e.printStackTrace(); throw new IllegalStateException("should not be happening"); } catch (JsonMappingException e) { e.printStackTrace(); throw new IllegalStateException("should not be happening"); } catch (IOException e) { e.printStackTrace(); throw new IllegalStateException("should not be happening"); } } toBeResolved = moreToResolve; } }
From source file:snpviewer.SnpViewer.java
public void writeRegionToFile(final String chromosome, final double start, final double end) { /* get coordinates of selection and report back * write SNPs in region to file//from w w w.j ava2 s .c o m */ FileChooser fileChooser = new FileChooser(); FileChooser.ExtensionFilter extFilter = new FileChooser.ExtensionFilter("Excel (*.xlsx)", "*.xlsx"); fileChooser.getExtensionFilters().add(extFilter); fileChooser.setTitle("Write region to Excel file (.xlsx)..."); File rFile = fileChooser.showSaveDialog(mainWindow); if (rFile == null) { return; } else if (!rFile.getName().endsWith(".xlsx")) { rFile = new File(rFile.getAbsolutePath() + ".xlsx"); } final File regionFile = rFile; final Task<Boolean> writeTask = new Task() { @Override protected Boolean call() throws Exception { try { updateProgress(-1, -1); ArrayList<SnpFile> bothFiles = new ArrayList<>(); bothFiles.addAll(affFiles); bothFiles.addAll(unFiles); TreeMap<Integer, HashMap<String, String>> coordMap = new TreeMap(); /*coordmap - key is position, key of hashmap * is input filename and value call */ HashMap<Integer, String> coordToId = new HashMap<>(); double progress = 0; double total = bothFiles.size() * 5; try { BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(regionFile)); Workbook wb = new XSSFWorkbook(); Sheet sheet = wb.createSheet(); int rowNo = 0; Row row = sheet.createRow(rowNo++); for (SnpFile f : bothFiles) { if (isCancelled()) { return false; } updateProgress(++progress, total); updateMessage("Reading region in " + f.inputFile.getName()); List<SnpFile.SnpLine> lines = f.getSnpsInRegion(chromosome, (int) start, (int) end); for (SnpFile.SnpLine snpLine : lines) { if (isCancelled()) { return false; } Integer coord = snpLine.getPosition(); if (!coordMap.containsKey(coord)) { coordMap.put(coord, new HashMap<String, String>()); } String filename = f.inputFile.getName(); String rsId = snpLine.getId(); String call = snpLine.getCall(); coordMap.get(coord).put(filename, call); coordToId.put(coord, rsId); } } Cell cell = row.createCell(0); cell.setCellValue( "chr" + chromosome + ":" + coordMap.firstKey() + "-" + coordMap.lastKey()); row = sheet.createRow(rowNo++); cell = row.createCell(0); cell.setCellValue( coordToId.get(coordMap.firstKey()) + ";" + coordToId.get(coordMap.lastKey())); row = sheet.createRow(rowNo++); int colNo = 0; cell = row.createCell(colNo++); cell.setCellValue("Position"); cell = row.createCell(colNo++); cell.setCellValue("rsID"); for (SnpFile f : bothFiles) { cell = row.createCell(colNo++); if (f.getSampleName() != null && f.getSampleName().length() > 0) { cell.setCellValue(f.getSampleName()); } else { cell.setCellValue(f.getInputFileName()); } } progress = coordMap.size(); total = 5 * coordMap.size(); updateMessage("Writing region to file..."); for (Entry current : coordMap.entrySet()) { if (isCancelled()) { return false; } progress += 4; updateProgress(progress, total); row = sheet.createRow(rowNo++); colNo = 0; Integer coord = (Integer) current.getKey(); cell = row.createCell(colNo++); cell.setCellValue(coord); String rsId = coordToId.get(coord); cell = row.createCell(colNo++); cell.setCellValue(rsId); HashMap<String, String> fileToCall = (HashMap<String, String>) current.getValue(); for (SnpFile f : bothFiles) { cell = row.createCell(colNo++); if (fileToCall.containsKey(f.inputFile.getName())) { cell.setCellValue(fileToCall.get(f.inputFile.getName())); } else { cell.setCellValue("-"); } } } CellRangeAddress[] regions = { new CellRangeAddress(0, rowNo, 2, 2 + bothFiles.size()) }; SheetConditionalFormatting sheetCF = sheet.getSheetConditionalFormatting(); ConditionalFormattingRule rule1 = sheetCF .createConditionalFormattingRule(ComparisonOperator.EQUAL, "\"AA\""); PatternFormatting fill1 = rule1.createPatternFormatting(); fill1.setFillBackgroundColor(IndexedColors.LIGHT_GREEN.index); fill1.setFillPattern(PatternFormatting.SOLID_FOREGROUND); ConditionalFormattingRule rule2 = sheetCF .createConditionalFormattingRule(ComparisonOperator.EQUAL, "\"BB\""); PatternFormatting fill2 = rule2.createPatternFormatting(); fill2.setFillBackgroundColor(IndexedColors.PALE_BLUE.index); fill2.setFillPattern(PatternFormatting.SOLID_FOREGROUND); ConditionalFormattingRule rule3 = sheetCF .createConditionalFormattingRule(ComparisonOperator.EQUAL, "\"AB\""); PatternFormatting fill3 = rule3.createPatternFormatting(); fill3.setFillBackgroundColor(IndexedColors.ROSE.index); fill3.setFillPattern(PatternFormatting.SOLID_FOREGROUND); sheetCF.addConditionalFormatting(regions, rule3, rule2); sheetCF.addConditionalFormatting(regions, rule1); wb.write(out); out.close(); return true; } catch (IOException ex) { return false; } } catch (Exception ex) { return false; } } };//end of task setProgressMode(true); progressBar.progressProperty().bind(writeTask.progressProperty()); progressMessage.textProperty().bind(writeTask.messageProperty()); writeTask.setOnSucceeded(new EventHandler<WorkerStateEvent>() { @Override public void handle(WorkerStateEvent e) { if (e.getSource().getValue() == true) { Dialogs.showInformationDialog(null, "Region written to file " + "(" + regionFile.getName() + ") successfully", "Region Written", "SNP Viewer"); } else { Dialogs.showErrorDialog(null, "Region write failed.", "Write Failed", "SNP Viewer"); } setProgressMode(false); progressBar.progressProperty().unbind(); progressBar.progressProperty().set(0); progressMessage.textProperty().unbind(); progressMessage.setText(""); progressTitle.setText(""); } }); writeTask.setOnFailed(new EventHandler<WorkerStateEvent>() { @Override public void handle(WorkerStateEvent e) { setProgressMode(false); progressBar.progressProperty().unbind(); progressBar.progressProperty().set(0); progressMessage.textProperty().unbind(); progressMessage.setText(""); progressTitle.setText("Region write failed!"); Dialogs.showErrorDialog(null, "Error writing region to file\n", "Region write error", "SNP Viewer", e.getSource().getException()); } }); writeTask.setOnCancelled(new EventHandler<WorkerStateEvent>() { @Override public void handle(WorkerStateEvent e) { progressMessage.setText("Region write cancelled"); progressTitle.setText("Cancelled"); setProgressMode(false); progressBar.progressProperty().unbind(); progressBar.progressProperty().set(0); Dialogs.showErrorDialog(null, "Error writing region to file\n", "Region write error", "SNP Viewer"); } }); cancelButton.setOnAction(new EventHandler<ActionEvent>() { @Override public void handle(ActionEvent actionEvent) { writeTask.cancel(); } }); progressTitle.setText("Writing region to .xlsx file"); new Thread(writeTask).start(); }