List of usage examples for java.util TreeSet add
public boolean add(E e)
From source file:net.spfbl.data.Block.java
public static String find(String userEmail, String token, boolean findDNSBL, boolean findREGEX, boolean findWHOIS, boolean autoBlock) { TreeSet<String> whoisSet = new TreeSet<String>(); LinkedList<String> regexList = new LinkedList<String>(); if (token == null) { return null; } else if (Domain.isEmail(token)) { String sender = token.toLowerCase(); int index1 = sender.indexOf('@'); int index2 = sender.lastIndexOf('@'); String part = sender.substring(0, index1 + 1); String senderDomain = sender.substring(index2); if (SET.contains(sender)) { return sender; } else if (userEmail != null && SET.contains(userEmail + ':' + sender)) { return userEmail + ':' + sender; } else if (SET.contains(part)) { return part; } else if (userEmail != null && SET.contains(userEmail + ':' + part)) { return userEmail + ':' + part; } else if (SET.contains(senderDomain)) { return senderDomain; } else if (userEmail != null && SET.contains(userEmail + ':' + senderDomain)) { return userEmail + ':' + senderDomain; } else {//from www.j av a 2s . c o m int index3 = senderDomain.length(); while ((index3 = senderDomain.lastIndexOf('.', index3 - 1)) > index2) { String subdomain = senderDomain.substring(0, index3 + 1); if (SET.contains(subdomain)) { return subdomain; } else if (userEmail != null && SET.contains(userEmail + ':' + subdomain)) { return userEmail + ':' + subdomain; } } String host = '.' + senderDomain.substring(1); do { int index = host.indexOf('.') + 1; host = host.substring(index); String token2 = '.' + host; if (SET.contains(token2)) { return token2; } else if (userEmail != null && SET.contains(userEmail + ':' + token2)) { return userEmail + ':' + token2; } regexList.addFirst(token2); } while (host.contains(".")); int index4 = sender.length(); while ((index4 = sender.lastIndexOf('.', index4 - 1)) > index2) { String subsender = sender.substring(0, index4 + 1); if (SET.contains(subsender)) { return subsender; } else if (userEmail != null && SET.contains(userEmail + ':' + subsender)) { return userEmail + ':' + subsender; } } } if (senderDomain.endsWith(".br")) { whoisSet.add(senderDomain); } regexList.add(sender); } else if (Subnet.isValidIP(token)) { token = Subnet.normalizeIP(token); String cidr; String dnsbl; if (SET.contains(token)) { return token; } else if (userEmail != null && SET.contains(userEmail + ':' + token)) { return userEmail + ':' + token; } else if ((cidr = CIDR.get(userEmail, token)) != null) { return cidr; } else if (findDNSBL && (dnsbl = DNSBL.get(userEmail, token)) != null) { return dnsbl; } Reverse reverse = Reverse.get(token); if (reverse != null) { for (String host : reverse.getAddressSet()) { String block = find(userEmail, host, findDNSBL, findREGEX, findWHOIS, autoBlock); if (block != null) { return block; } } } regexList.add(token); } else if (Domain.isHostname(token)) { token = Domain.normalizeHostname(token, true); String host = token; do { int index = host.indexOf('.') + 1; host = host.substring(index); String token2 = '.' + host; if (SET.contains(token2)) { return token2; } else if (userEmail != null && SET.contains(userEmail + ':' + token2)) { return userEmail + ':' + token2; } regexList.addFirst(token2); } while (host.contains(".")); if (token.endsWith(".br")) { whoisSet.add(token); } } else { regexList.add(token); } if (findREGEX) { try { // Verifica um critrio do REGEX. String regex; if ((regex = REGEX.get(userEmail, regexList, autoBlock)) != null) { return regex; } } catch (Exception ex) { Server.logError(ex); } } if (findWHOIS) { try { // Verifica critrios do WHOIS. String whois; if ((whois = WHOIS.get(userEmail, whoisSet, autoBlock)) != null) { return whois; } } catch (Exception ex) { Server.logError(ex); } } return null; }
From source file:de.tudarmstadt.ukp.experiments.argumentation.convincingness.sampling.Step6GraphTransitivityCleaner.java
public GraphCleaningResults processSingleFile(File file, File outputDir, String prefix, Boolean collectGeneratedArgumentPairs) throws Exception { GraphCleaningResults result = new GraphCleaningResults(); File outFileTable = new File(outputDir, prefix + file.getName() + "_table.csv"); File outFileInfo = new File(outputDir, prefix + file.getName() + "_info.txt"); PrintStream psTable = new PrintStream(new FileOutputStream(outFileTable)); PrintStream psInfo = new PrintStream(new FileOutputStream(outFileInfo)); // load one topic/side List<AnnotatedArgumentPair> pairs = new ArrayList<>( (List<AnnotatedArgumentPair>) XStreamTools.getXStream().fromXML(file)); int fullDataSize = pairs.size(); // filter out missing gold data Iterator<AnnotatedArgumentPair> iterator = pairs.iterator(); while (iterator.hasNext()) { AnnotatedArgumentPair pair = iterator.next(); if (pair.getGoldLabel() == null) { iterator.remove();//from w w w .j ava 2s. c o m } // or we want to completely remove equal edges in advance! else if (this.removeEqualEdgesParam && "equal".equals(pair.getGoldLabel())) { iterator.remove(); } } // sort pairs by their weight this.argumentPairListSorter.sortArgumentPairs(pairs); int preFilteredDataSize = pairs.size(); // compute correlation between score threshold and number of removed edges double[] correlationEdgeWeights = new double[pairs.size()]; double[] correlationRemovedEdges = new double[pairs.size()]; // only cycles of length 0 to 5 are interesting (5+ are too big) Range<Integer> range = Range.between(0, 5); psTable.print( "EdgeWeightThreshold\tPairs\tignoredEdgesCount\tIsDAG\tTransitivityScoreMean\tTransitivityScoreMax\tTransitivityScoreSamples\tEdges\tNodes\t"); for (int j = range.getMinimum(); j <= range.getMaximum(); j++) { psTable.print("Cycles_" + j + "\t"); } psTable.println(); // store the indices of all pairs (edges) that have been successfully added without // generating cycles TreeSet<Integer> addedPairsIndices = new TreeSet<>(); // number of edges ignored as they generated cycles int ignoredEdgesCount = 0; Graph lastGraph = null; // flag that the first cycle was already processed boolean firstCycleAlreadyHit = false; for (int i = 1; i < pairs.size(); i++) { // now filter the finalArgumentPairList and add only pairs that have not generated cycles List<AnnotatedArgumentPair> subList = new ArrayList<>(); for (Integer index : addedPairsIndices) { subList.add(pairs.get(index)); } // and add the current at the end subList.add(pairs.get(i)); // what is the current lowest value of a pair weight? double weakestEdgeWeight = computeEdgeWeight(subList.get(subList.size() - 1), LAMBDA_PENALTY); // Graph graph = buildGraphFromArgumentPairs(finalArgumentPairList); int numberOfLoops; // map for storing cycles by their length TreeMap<Integer, TreeSet<String>> lengthCyclesMap = new TreeMap<>(); Graph graph = buildGraphFromArgumentPairs(subList); lastGraph = graph; List<List<Object>> cyclesInGraph = findCyclesInGraph(graph); DescriptiveStatistics transitivityScore = new DescriptiveStatistics(); if (cyclesInGraph.isEmpty()) { // we have DAG transitivityScore = computeTransitivityScores(graph); // update results result.maxTransitivityScore = (int) transitivityScore.getMax(); result.avgTransitivityScore = transitivityScore.getMean(); } numberOfLoops = cyclesInGraph.size(); // initialize map for (int r = range.getMinimum(); r <= range.getMaximum(); r++) { lengthCyclesMap.put(r, new TreeSet<String>()); } // we hit a loop if (numberOfLoops > 0) { // let's update the result if (!firstCycleAlreadyHit) { result.graphSizeEdgesBeforeFirstCycle = graph.getEdgeCount(); result.graphSizeNodesBeforeFirstCycle = graph.getNodeCount(); // find the shortest cycle int shortestCycleLength = Integer.MAX_VALUE; for (List<Object> cycle : cyclesInGraph) { shortestCycleLength = Math.min(shortestCycleLength, cycle.size()); } result.lengthOfFirstCircle = shortestCycleLength; result.pairsBeforeFirstCycle = i; firstCycleAlreadyHit = true; } // ignore this edge further ignoredEdgesCount++; // update counts of different cycles lengths for (List<Object> cycle : cyclesInGraph) { int currentSize = cycle.size(); // convert to sorted set of nodes List<String> cycleAsSortedIDs = new ArrayList<>(); for (Object o : cycle) { cycleAsSortedIDs.add(o.toString()); } Collections.sort(cycleAsSortedIDs); if (range.contains(currentSize)) { lengthCyclesMap.get(currentSize).add(cycleAsSortedIDs.toString()); } } } else { addedPairsIndices.add(i); } // we hit the first cycle // collect loop sizes StringBuilder loopsAsString = new StringBuilder(); for (int j = range.getMinimum(); j <= range.getMaximum(); j++) { // loopsAsString.append(j).append(":"); loopsAsString.append(lengthCyclesMap.get(j).size()); loopsAsString.append("\t"); } psTable.printf(Locale.ENGLISH, "%.4f\t%d\t%d\t%b\t%.2f\t%d\t%d\t%d\t%d\t%s%n", weakestEdgeWeight, i, ignoredEdgesCount, numberOfLoops == 0, Double.isNaN(transitivityScore.getMean()) ? 0d : transitivityScore.getMean(), (int) transitivityScore.getMax(), transitivityScore.getN(), graph.getEdgeCount(), graph.getNodeCount(), loopsAsString.toString().trim()); // update result result.finalGraphSizeEdges = graph.getEdgeCount(); result.finalGraphSizeNodes = graph.getNodeCount(); result.ignoredEdgesThatBrokeDAG = ignoredEdgesCount; // update stats for correlation correlationEdgeWeights[i] = weakestEdgeWeight; // correlationRemovedEdges[i] = (double) ignoredEdgesCount; // let's try: if we keep = 0, if we remove = 1 correlationRemovedEdges[i] = numberOfLoops == 0 ? 0.0 : 1.0; } psInfo.println("Original: " + fullDataSize + ", removed by MACE: " + (fullDataSize - preFilteredDataSize) + ", final: " + (preFilteredDataSize - ignoredEdgesCount) + " (removed: " + ignoredEdgesCount + ")"); double[][] matrix = new double[correlationEdgeWeights.length][]; for (int i = 0; i < correlationEdgeWeights.length; i++) { matrix[i] = new double[2]; matrix[i][0] = correlationEdgeWeights[i]; matrix[i][1] = correlationRemovedEdges[i]; } PearsonsCorrelation pearsonsCorrelation = new PearsonsCorrelation(matrix); double pValue = pearsonsCorrelation.getCorrelationPValues().getEntry(0, 1); double correlation = pearsonsCorrelation.getCorrelationMatrix().getEntry(0, 1); psInfo.printf(Locale.ENGLISH, "Correlation: %.3f, p-Value: %.4f%n", correlation, pValue); if (lastGraph == null) { throw new IllegalStateException("Graph is null"); } // close psInfo.close(); psTable.close(); // save filtered final gold data List<AnnotatedArgumentPair> finalArgumentPairList = new ArrayList<>(); for (Integer index : addedPairsIndices) { finalArgumentPairList.add(pairs.get(index)); } XStreamTools.toXML(finalArgumentPairList, new File(outputDir, prefix + file.getName())); // TODO: here, we can add newly generated edges from graph transitivity if (collectGeneratedArgumentPairs) { Set<GeneratedArgumentPair> generatedArgumentPairs = new HashSet<>(); // collect all arguments Map<String, Argument> allArguments = new HashMap<>(); for (ArgumentPair argumentPair : pairs) { allArguments.put(argumentPair.getArg1().getId(), argumentPair.getArg1()); allArguments.put(argumentPair.getArg2().getId(), argumentPair.getArg2()); } Graph finalGraph = buildGraphFromArgumentPairs(finalArgumentPairList); for (Edge e : finalGraph.getEdgeSet()) { e.setAttribute(WEIGHT, 1.0); } for (Node j : finalGraph) { for (Node k : finalGraph) { if (j != k) { // is there a path between? BellmanFord bfShortest = new BellmanFord(WEIGHT, j.getId()); bfShortest.init(finalGraph); bfShortest.compute(); Path shortestPath = bfShortest.getShortestPath(k); if (shortestPath.size() > 0) { // we have a path GeneratedArgumentPair ap = new GeneratedArgumentPair(); Argument arg1 = allArguments.get(j.getId()); if (arg1 == null) { throw new IllegalStateException("Cannot find argument " + j.getId()); } ap.setArg1(arg1); Argument arg2 = allArguments.get(k.getId()); if (arg2 == null) { throw new IllegalStateException("Cannot find argument " + k.getId()); } ap.setArg2(arg2); ap.setGoldLabel("a1"); generatedArgumentPairs.add(ap); } } } } // and now add the reverse ones Set<GeneratedArgumentPair> generatedReversePairs = new HashSet<>(); for (GeneratedArgumentPair pair : generatedArgumentPairs) { GeneratedArgumentPair ap = new GeneratedArgumentPair(); ap.setArg1(pair.getArg2()); ap.setArg2(pair.getArg1()); ap.setGoldLabel("a2"); generatedReversePairs.add(ap); } generatedArgumentPairs.addAll(generatedReversePairs); // and save it XStreamTools.toXML(generatedArgumentPairs, new File(outputDir, "generated_" + prefix + file.getName())); } result.fullPairsSize = fullDataSize; result.removedApriori = (fullDataSize - preFilteredDataSize); result.finalPairsRetained = finalArgumentPairList.size(); // save the final graph Graph outGraph = cleanCopyGraph(lastGraph); FileSinkDGS dgs1 = new FileSinkDGS(); File outFile = new File(outputDir, prefix + file.getName() + ".dgs"); System.out.println("Saved to " + outFile); FileWriter w1 = new FileWriter(outFile); dgs1.writeAll(outGraph, w1); w1.close(); return result; }
From source file:org.dasein.cloud.aws.compute.EC2Instance.java
private Set<Metric> calculate(String metric, String unit, String id, boolean idIsVolumeId, long startTimestamp, long endTimestamp) throws CloudException, InternalException { APITrace.begin(getProvider(), "calculateVMAnalytics"); try {/*w w w.java 2 s .co m*/ if (!getProvider().getEC2Provider().isAWS()) { return new TreeSet<Metric>(); } Map<String, String> parameters = getProvider().getStandardCloudWatchParameters(getContext(), EC2Method.GET_METRIC_STATISTICS); SimpleDateFormat fmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"); fmt.setCalendar(UTC_CALENDAR); EC2Method method; NodeList blocks; Document doc; parameters.put("EndTime", fmt.format(new Date(endTimestamp))); parameters.put("StartTime", fmt.format(new Date(startTimestamp))); parameters.put("MetricName", metric); parameters.put("Namespace", idIsVolumeId ? "AWS/EBS" : "AWS/EC2"); parameters.put("Unit", unit); parameters.put("Dimensions.member.Name.1", idIsVolumeId ? "VolumeId" : "InstanceId"); parameters.put("Dimensions.member.Value.1", id); parameters.put("Statistics.member.1", "Average"); parameters.put("Statistics.member.2", "Minimum"); parameters.put("Statistics.member.3", "Maximum"); parameters.put("Period", "60"); method = new EC2Method("monitoring", getProvider(), parameters); try { doc = method.invoke(); } catch (EC2Exception e) { logger.error(e.getSummary()); throw new CloudException(e); } TreeSet<Metric> metrics = new TreeSet<Metric>(); fmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); fmt.setCalendar(UTC_CALENDAR); blocks = doc.getElementsByTagName("member"); for (int i = 0; i < blocks.getLength(); i++) { NodeList items = blocks.item(i).getChildNodes(); Metric m = new Metric(); for (int j = 0; j < items.getLength(); j++) { Node item = items.item(j); if (item.getNodeName().equals("Timestamp")) { String dateString = item.getFirstChild().getNodeValue(); try { m.timestamp = fmt.parse(dateString).getTime(); } catch (ParseException e) { logger.error(e); throw new InternalException(e); } } else if (item.getNodeName().equals("Average")) { m.average = Double.parseDouble(item.getFirstChild().getNodeValue()); } else if (item.getNodeName().equals("Minimum")) { m.minimum = Double.parseDouble(item.getFirstChild().getNodeValue()); } else if (item.getNodeName().equals("Maximum")) { m.maximum = Double.parseDouble(item.getFirstChild().getNodeValue()); } else if (item.getNodeName().equals("Samples")) { m.samples = (int) Double.parseDouble(item.getFirstChild().getNodeValue()); } } metrics.add(m); } return metrics; } finally { APITrace.end(); } }
From source file:org.dasein.cloud.vcloud.vCloudMethod.java
private @Nonnull Version getVersion() throws CloudException, InternalException { Cache<Version> cache = Cache.getInstance(provider, "vCloudVersions", Version.class, CacheLevel.CLOUD, new TimePeriod<Day>(1, TimePeriod.DAY)); ProviderContext ctx = provider.getContext(); if (ctx == null) { throw new CloudException("No context was defined for this request"); }//from w w w . j a v a2 s . c o m { Iterable<Version> versions = cache.get(ctx); Iterator<Version> it = (versions == null ? null : versions.iterator()); if (it != null && it.hasNext()) { return it.next(); } } // TODO: how does vCHS do version discovery? if (ctx.getCloud().getEndpoint().startsWith("https://vchs")) { // This is a complete hack that needs to be changed to reflect vCHS version discovery Version version = new Version(); version.loginUrl = ctx.getCloud().getEndpoint() + "/api/vchs/sessions"; version.version = "5.6"; cache.put(ctx, Collections.singletonList(version)); return version; } if (wire.isDebugEnabled()) { wire.debug(""); wire.debug(">>> [GET (" + (new Date()) + ")] -> " + ctx.getCloud().getEndpoint() + " >--------------------------------------------------------------------------------------"); } try { final String[] preferred = provider.getVersionPreference(); HttpClient client = getClient(false); HttpGet method = new HttpGet(ctx.getCloud().getEndpoint() + "/api/versions"); if (wire.isDebugEnabled()) { wire.debug(method.getRequestLine().toString()); for (Header header : method.getAllHeaders()) { wire.debug(header.getName() + ": " + header.getValue()); } wire.debug(""); } HttpResponse response; StatusLine status; try { APITrace.trace(provider, "GET versions"); response = client.execute(method); if (wire.isDebugEnabled()) { wire.debug(response.getStatusLine().toString()); for (Header header : response.getAllHeaders()) { wire.debug(header.getName() + ": " + header.getValue()); } wire.debug(""); } status = response.getStatusLine(); } catch (IOException e) { throw new CloudException(e); } if (status.getStatusCode() == HttpServletResponse.SC_OK) { HttpEntity entity = response.getEntity(); String body; try { body = EntityUtils.toString(entity); if (wire.isDebugEnabled()) { wire.debug(body); wire.debug(""); } } catch (IOException e) { throw new CloudException(CloudErrorType.GENERAL, status.getStatusCode(), status.getReasonPhrase(), e.getMessage()); } try { ByteArrayInputStream bas = new ByteArrayInputStream(body.getBytes()); DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); DocumentBuilder parser = factory.newDocumentBuilder(); Document doc = parser.parse(bas); bas.close(); NodeList versions = doc.getElementsByTagName("VersionInfo"); TreeSet<Version> set = new TreeSet<Version>(new Comparator<Version>() { public int compare(Version version1, Version version2) { if (version1.equals(version2)) { return 0; } if (preferred != null) { for (String v : preferred) { if (v.equals(version1.version)) { return -1; } else if (v.equals(version2.version)) { return 1; } } } for (String v : VERSIONS) { if (v.equals(version1.version)) { return -1; } else if (v.equals(version2.version)) { return 1; } } return -version1.version.compareTo(version2.version); } }); for (int i = 0; i < versions.getLength(); i++) { Node versionInfo = versions.item(i); NodeList vattrs = versionInfo.getChildNodes(); String version = null; String url = null; for (int j = 0; j < vattrs.getLength(); j++) { Node attr = vattrs.item(j); if (attr.getNodeName().equalsIgnoreCase("Version") && attr.hasChildNodes()) { version = attr.getFirstChild().getNodeValue().trim(); } else if (attr.getNodeName().equalsIgnoreCase("LoginUrl") && attr.hasChildNodes()) { url = attr.getFirstChild().getNodeValue().trim(); } } if (version == null || url == null || !isSupported(version)) { continue; } Version v = new Version(); v.version = version; v.loginUrl = url; set.add(v); } if (set.isEmpty()) { throw new CloudException("Unable to identify a supported version"); } Version v = set.iterator().next(); cache.put(ctx, set); return v; } catch (IOException e) { throw new CloudException(CloudErrorType.GENERAL, status.getStatusCode(), status.getReasonPhrase(), e.getMessage()); } catch (ParserConfigurationException e) { throw new CloudException(CloudErrorType.GENERAL, status.getStatusCode(), status.getReasonPhrase(), e.getMessage()); } catch (SAXException e) { throw new CloudException(CloudErrorType.GENERAL, status.getStatusCode(), status.getReasonPhrase(), e.getMessage()); } } else { logger.error("Expected OK for GET request, got " + status.getStatusCode()); String xml = null; try { HttpEntity entity = response.getEntity(); if (entity != null) { xml = EntityUtils.toString(entity); if (wire.isDebugEnabled()) { wire.debug(xml); wire.debug(""); } } } catch (IOException e) { logger.error("Failed to read response error due to a cloud I/O error: " + e.getMessage()); throw new CloudException(e); } vCloudException.Data data = null; if (xml != null && !xml.equals("")) { Document doc = parseXML(xml); String docElementTagName = doc.getDocumentElement().getTagName(); String nsString = ""; if (docElementTagName.contains(":")) nsString = docElementTagName.substring(0, docElementTagName.indexOf(":") + 1); NodeList errors = doc.getElementsByTagName(nsString + "Error"); if (errors.getLength() > 0) { data = vCloudException.parseException(status.getStatusCode(), errors.item(0)); } } if (data == null) { throw new vCloudException(CloudErrorType.GENERAL, status.getStatusCode(), response.getStatusLine().getReasonPhrase(), "No further information"); } logger.error("[" + status.getStatusCode() + " : " + data.title + "] " + data.description); throw new vCloudException(data); } } finally { if (wire.isDebugEnabled()) { wire.debug("<<< [GET (" + (new Date()) + ")] -> " + ctx.getEndpoint() + " <--------------------------------------------------------------------------------------"); wire.debug(""); } } }
From source file:asterix.parser.classad.ClassAd.java
public boolean privateGetInternalReferences(ExprTree expr, ClassAd ad, EvalState state, TreeSet<String> refs, boolean fullNames) throws HyracksDataException { switch (expr.getKind()) { //nothing to be found here! case LITERAL_NODE: { return true; }/* ww w .ja va 2 s .co m*/ case ATTRREF_NODE: { ClassAd start = new ClassAd(); ExprTreeHolder tree = new ExprTreeHolder(); ExprTreeHolder result = new ExprTreeHolder(); AMutableCharArrayString attr = new AMutableCharArrayString(); Value val = new Value(); MutableBoolean abs = new MutableBoolean(); ((AttributeReference) expr).getComponents(tree, attr, abs); //figuring out which state to base this off of if (tree.getInnerTree() == null) { start = abs.booleanValue() ? state.getRootAd() : state.getCurAd(); //remove circularity if (abs.booleanValue() && (start == null)) { return false; } } else { boolean orig_inAttrRefScope = state.isInAttrRefScope(); state.setInAttrRefScope(true); boolean rv = privateGetInternalReferences(tree, ad, state, refs, fullNames); state.setInAttrRefScope(orig_inAttrRefScope); if (!rv) { return false; } if (!tree.publicEvaluate(state, val)) { return false; } // TODO Do we need extra handling for list values? // Should types other than undefined, error, or list // cause a failure? if (val.isUndefinedValue()) { return true; } //otherwise, if the tree didn't evaluate to a classad, //we have a problemo, mon. //TODO: but why? if (!val.isClassAdValue(start)) { return false; } } ClassAd curAd = state.getCurAd(); switch (start.lookupInScope(attr.toString(), result, state)) { case EVAL_ERROR_Int: return false; //attr is external, so let's find the internals in that //result //JUST KIDDING case EVAL_UNDEF_Int: { //boolean rval = _GetInternalReferences(result, ad, state, refs, fullNames); //state.getCurAd() = curAd; return true; } case EVAL_OK_Int: { //whoo, it's internal. // Check whether the attribute was found in the root // ad for this evaluation and that the attribute isn't // one of our special ones (self, parent, my, etc.). // If the ad actually has an attribute with the same // name as one of our special attributes, then count // that as an internal reference. // TODO LookupInScope() knows whether it's returning // the expression of one of the special attributes // or that of an attribute that actually appears in // the ad. If it told us which one, then we could // avoid the Lookup() call below. if (state.getCurAd() == state.getRootAd() && state.getCurAd().lookup(attr.toString()) != null) { refs.add(attr.toString()); } if (state.getDepthRemaining() <= 0) { state.setCurAd(curAd); return false; } state.decrementDepth(); boolean rval = privateGetInternalReferences(result, ad, state, refs, fullNames); state.incrementDepth(); //TODO: Does this actually matter? state.setCurAd(curAd); return rval; } case EVAL_FAIL_Int: default: // "enh??" return false; } } case OP_NODE: { //recurse on subtrees AMutableInt32 op = new AMutableInt32(0); ExprTreeHolder t1 = new ExprTreeHolder(); ExprTreeHolder t2 = new ExprTreeHolder(); ExprTreeHolder t3 = new ExprTreeHolder(); ((Operation) expr).getComponents(op, t1, t2, t3); if (t1.getInnerTree() != null && !privateGetInternalReferences(t1, ad, state, refs, fullNames)) { return false; } if (t2.getInnerTree() != null && !privateGetInternalReferences(t2, ad, state, refs, fullNames)) { return false; } if (t3.getInnerTree() != null && !privateGetInternalReferences(t3, ad, state, refs, fullNames)) { return false; } return true; } case FN_CALL_NODE: { //recurse on the subtrees! AMutableCharArrayString fnName = new AMutableCharArrayString(); ExprList args = new ExprList(); ((FunctionCall) expr).getComponents(fnName, args); for (ExprTree exprTree : args.getExprList()) { if (!privateGetInternalReferences(exprTree, ad, state, refs, fullNames)) { return false; } } return true; } case CLASSAD_NODE: { //also recurse on subtrees... HashMap<CaseInsensitiveString, ExprTree> attrs = new HashMap<CaseInsensitiveString, ExprTree>(); // If this ClassAd is only being used here as the scoping // for an attribute reference, don't recurse into all of // its attributes. if (state.isInAttrRefScope()) { return true; } ((ClassAd) expr).getComponents(attrs); for (Entry<CaseInsensitiveString, ExprTree> entry : attrs.entrySet()) { if (state.getDepthRemaining() <= 0) { return false; } state.decrementDepth(); boolean ret = privateGetInternalReferences(entry.getValue(), ad, state, refs, fullNames); state.incrementDepth(); if (!ret) { return false; } } return true; } case EXPR_LIST_NODE: { ExprList exprs = new ExprList(); ((ExprList) expr).getComponents(exprs); for (ExprTree exprTree : exprs.getExprList()) { if (state.getDepthRemaining() <= 0) { return false; } state.decrementDepth(); boolean ret = privateGetInternalReferences(exprTree, ad, state, refs, fullNames); state.incrementDepth(); if (!ret) { return false; } } return true; } default: return false; } }
From source file:ddf.catalog.source.solr.SolrProviderTest.java
@Test public void testStartIndexWithSorting() throws Exception { deleteAllIn(provider);/*from w ww . ja va 2 s .c o m*/ List<Metacard> metacards = new ArrayList<Metacard>(); DateTime dt = new DateTime(1985, 1, 1, 1, 1, 1, 1, DateTimeZone.UTC); TreeSet<Date> calculatedDates = new TreeSet<Date>(); for (int j = 0; j < 10; j++) { for (int i = 0; i < 100; i = i + 10) { MetacardImpl metacard = new MockMetacard(Library.getFlagstaffRecord()); // ingest sporadically the effective dates so the default return // order won't be ordered Date calculatedDate = dt.plusDays(100 - i + 10 - j).toDate(); calculatedDates.add(calculatedDate); metacard.setEffectiveDate(calculatedDate); metacards.add(metacard); } } // The TreeSet will sort them, the array will give me access to everyone // without an iterator Date[] dates = new Date[calculatedDates.size()]; calculatedDates.toArray(dates); /** CREATE **/ CreateResponse response = create(metacards); LOGGER.info("CREATED {} records.", response.getCreatedMetacards().size()); CommonQueryBuilder queryBuilder = new CommonQueryBuilder(); QueryImpl query = queryBuilder.queryByProperty(Metacard.CONTENT_TYPE, MockMetacard.DEFAULT_TYPE); int maxSize = 20; int startIndex = 2; // STARTINDEX=2, MAXSIZE=20 query.setPageSize(maxSize); query.setStartIndex(startIndex); SortByImpl sortBy = new SortByImpl(queryBuilder.filterFactory.property(Metacard.EFFECTIVE), org.opengis.filter.sort.SortOrder.ASCENDING); query.setSortBy(sortBy); SourceResponse sourceResponse = provider.query(new QueryRequestImpl(query)); assertEquals(maxSize, sourceResponse.getResults().size()); for (int i = 0; i < sourceResponse.getResults().size(); i++) { Result r = sourceResponse.getResults().get(i); Date effectiveDate = r.getMetacard().getEffectiveDate(); DateTime currentDate = new DateTime(effectiveDate.getTime()); LOGGER.debug("Testing current index: {}", startIndex + i); assertEquals(new DateTime(dates[startIndex - 1 + i].getTime()).getDayOfYear(), currentDate.getDayOfYear()); } // STARTINDEX=20, MAXSIZE=5 // a match-all queryByProperty query = queryBuilder.queryByProperty(Metacard.CONTENT_TYPE, MockMetacard.DEFAULT_TYPE); maxSize = 5; startIndex = 20; query.setPageSize(maxSize); query.setStartIndex(startIndex); sortBy = new SortByImpl(queryBuilder.filterFactory.property(Metacard.EFFECTIVE), org.opengis.filter.sort.SortOrder.ASCENDING); query.setSortBy(sortBy); sourceResponse = provider.query(new QueryRequestImpl(query)); assertEquals(maxSize, sourceResponse.getResults().size()); for (int i = 0; i < sourceResponse.getResults().size(); i++) { Result r = sourceResponse.getResults().get(i); Date effectiveDate = r.getMetacard().getEffectiveDate(); DateTime currentDate = new DateTime(effectiveDate.getTime()); LOGGER.debug("Testing current index: {}", startIndex + i); assertEquals(new DateTime(dates[startIndex - 1 + i].getTime()).getDayOfYear(), currentDate.getDayOfYear()); } // STARTINDEX=80, MAXSIZE=20 // a match-all queryByProperty query = queryBuilder.queryByProperty(Metacard.CONTENT_TYPE, MockMetacard.DEFAULT_TYPE); maxSize = 20; startIndex = 80; query.setPageSize(maxSize); query.setStartIndex(startIndex); sortBy = new SortByImpl(queryBuilder.filterFactory.property(Metacard.EFFECTIVE), org.opengis.filter.sort.SortOrder.ASCENDING); query.setSortBy(sortBy); sourceResponse = provider.query(new QueryRequestImpl(query)); assertEquals(maxSize, sourceResponse.getResults().size()); for (int i = 0; i < sourceResponse.getResults().size(); i++) { Result r = sourceResponse.getResults().get(i); Date effectiveDate = r.getMetacard().getEffectiveDate(); DateTime currentDate = new DateTime(effectiveDate.getTime()); LOGGER.debug("Testing current index: {}", startIndex + i); assertEquals(new DateTime(dates[startIndex - 1 + i].getTime()).getDayOfYear(), currentDate.getDayOfYear()); } // STARTINDEX=1, MAXSIZE=100 // a match-all queryByProperty query = queryBuilder.queryByProperty(Metacard.CONTENT_TYPE, MockMetacard.DEFAULT_TYPE); maxSize = 100; startIndex = 1; query.setPageSize(maxSize); query.setStartIndex(startIndex); sortBy = new SortByImpl(queryBuilder.filterFactory.property(Metacard.EFFECTIVE), org.opengis.filter.sort.SortOrder.ASCENDING); query.setSortBy(sortBy); sourceResponse = provider.query(new QueryRequestImpl(query)); assertEquals(maxSize, sourceResponse.getResults().size()); for (int i = 0; i < sourceResponse.getResults().size(); i++) { Result r = sourceResponse.getResults().get(i); Date effectiveDate = r.getMetacard().getEffectiveDate(); DateTime currentDate = new DateTime(effectiveDate.getTime()); LOGGER.debug("Testing current index: {}", startIndex + i); assertEquals(new DateTime(dates[startIndex - 1 + i].getTime()).getDayOfYear(), currentDate.getDayOfYear()); } }
From source file:hudson.plugins.project_inheritance.projects.InheritanceProject.java
/** * we are interested in getting the children ordered by last build time * if last build time exists/* w w w .j ava 2s. c om*/ */ public Collection<InheritanceProject> getChildrenByBuildDate( Map<InheritanceProject, Relationship> relationships) { //Using a TreeSet to do the sorting for last-build-time for us TreeSet<InheritanceProject> tree = new TreeSet<InheritanceProject>(new RunTimeComparator()); Map<InheritanceProject, Relationship> relations = this.getRelationships(); if (relations.isEmpty()) { return tree; } //Filtering for buildable children for (Map.Entry<InheritanceProject, Relationship> pair : relations.entrySet()) { InheritanceProject child = pair.getKey(); Relationship.Type type = pair.getValue().type; //Excluding non-childs if (type != Relationship.Type.CHILD) { continue; } //The child is buildable, so add it to the tree tree.add(child); } return tree; }
From source file:com.eucalyptus.objectstorage.WalrusManager.java
public ListBucketResponseType listBucket(ListBucketType request) throws EucalyptusCloudException { ListBucketResponseType reply = (ListBucketResponseType) request.getReply(); EntityWrapper<BucketInfo> db = EntityWrapper.get(BucketInfo.class); try {/*from ww w . j a v a 2s . c o m*/ String bucketName = request.getBucket(); BucketInfo bucketInfo = new BucketInfo(bucketName); bucketInfo.setHidden(false); List<BucketInfo> bucketList = db.queryEscape(bucketInfo); Context ctx = Contexts.lookup(); Account account = ctx.getAccount(); int maxKeys = -1; String maxKeysString = request.getMaxKeys(); if (maxKeysString != null) { maxKeys = Integer.parseInt(maxKeysString); if (maxKeys < 0) { throw new InvalidArgumentException("max-keys", "Argument max-keys must be an integer between 0 and " + Integer.MAX_VALUE); } } else { maxKeys = WalrusProperties.MAX_KEYS; } if (bucketList.size() > 0) { BucketInfo bucket = bucketList.get(0); BucketLogData logData = bucket.getLoggingEnabled() ? request.getLogData() : null; if (ctx.hasAdministrativePrivileges() || (bucket.canRead(account.getAccountNumber()) && (bucket.isGlobalRead() || Lookups.checkPrivilege(PolicySpec.S3_LISTBUCKET, PolicySpec.VENDOR_S3, PolicySpec.S3_RESOURCE_BUCKET, bucketName, null)))) { if (logData != null) { updateLogData(bucket, logData); reply.setLogData(logData); } if (Contexts.lookup().hasAdministrativePrivileges()) { try { if (bucketHasSnapshots(bucketName)) { db.rollback(); throw new NoSuchBucketException(bucketName); } } catch (Exception e) { db.rollback(); throw new EucalyptusCloudException(e); } } String prefix = request.getPrefix(); String delimiter = request.getDelimiter(); String marker = request.getMarker(); reply.setName(bucketName); reply.setIsTruncated(false); reply.setPrefix(prefix); reply.setMarker(marker); reply.setDelimiter(delimiter); reply.setMaxKeys(maxKeys); if (maxKeys == 0) { // No keys requested, so just return reply.setContents(new ArrayList<ListEntry>()); reply.setCommonPrefixesList(new ArrayList<CommonPrefixesEntry>()); db.commit(); return reply; } final int queryStrideSize = maxKeys + 1; EntityWrapper<ObjectInfo> dbObject = db.recast(ObjectInfo.class); ObjectInfo searchObj = new ObjectInfo(); searchObj.setBucketName(bucketName); searchObj.setLast(true); searchObj.setDeleted(false); Criteria objCriteria = dbObject.createCriteria(ObjectInfo.class); objCriteria.add(Example.create(searchObj)); objCriteria.addOrder(Order.asc("objectKey")); objCriteria.setMaxResults(queryStrideSize); // add one to, hopefully, indicate truncation in one call if (!Strings.isNullOrEmpty(marker)) { // The result set should be exclusive of the marker. marker could be a common prefix from a previous response. Look for keys that // lexicographically follow the marker and don't contain the marker as the prefix. objCriteria.add(Restrictions.gt("objectKey", marker)); } else { marker = ""; } if (!Strings.isNullOrEmpty(prefix)) { objCriteria.add(Restrictions.like("objectKey", prefix, MatchMode.START)); } else { prefix = ""; } // Ensure not null. if (Strings.isNullOrEmpty(delimiter)) { delimiter = ""; } List<ObjectInfo> objectInfos = null; int resultKeyCount = 0; ArrayList<ListEntry> contents = new ArrayList<ListEntry>(); // contents for reply String nextMarker = null; TreeSet<String> commonPrefixes = new TreeSet<String>(); int firstResult = -1; // Iterate over result sets of size maxkeys + 1 do { // Start listing from the 0th element and increment the first element to be listed by the query size objCriteria.setFirstResult(queryStrideSize * (++firstResult)); objectInfos = (List<ObjectInfo>) objCriteria.list(); if (objectInfos.size() > 0) { for (ObjectInfo objectInfo : objectInfos) { String objectKey = objectInfo.getObjectKey(); // Check if it will get aggregated as a commonprefix if (!Strings.isNullOrEmpty(delimiter)) { String[] parts = objectKey.substring(prefix.length()).split(delimiter); if (parts.length > 1) { String prefixString = prefix + parts[0] + delimiter; if (!StringUtils.equals(prefixString, marker) && !commonPrefixes.contains(prefixString)) { if (resultKeyCount == maxKeys) { // This is a new record, so we know we're truncating if this is true reply.setNextMarker(nextMarker); reply.setIsTruncated(true); resultKeyCount++; break; } commonPrefixes.add(prefixString); resultKeyCount++; // count the unique commonprefix as a single return entry // If max keys have been collected, set the next-marker. It might be needed for the response if the list is // truncated // If the common prefixes hit the limit set by max-keys, next-marker is the last common prefix if (resultKeyCount == maxKeys) { nextMarker = prefixString; } } continue; } } if (resultKeyCount == maxKeys) { // This is a new (non-commonprefix) record, so we know we're truncating reply.setNextMarker(nextMarker); reply.setIsTruncated(true); resultKeyCount++; break; } // Process the entry as a full key listing ListEntry listEntry = new ListEntry(); listEntry.setKey(objectKey); listEntry.setEtag(objectInfo.getEtag()); listEntry.setLastModified(DateUtils.format(objectInfo.getLastModified().getTime(), DateUtils.ALT_ISO8601_DATE_PATTERN)); listEntry.setStorageClass(objectInfo.getStorageClass()); listEntry.setSize(objectInfo.getSize()); listEntry.setStorageClass(objectInfo.getStorageClass()); try { Account ownerAccount = Accounts.lookupAccountById(objectInfo.getOwnerId()); listEntry.setOwner(new CanonicalUserType(ownerAccount.getCanonicalId(), ownerAccount.getName())); } catch (AuthException e) { db.rollback(); throw new AccessDeniedException("Bucket", bucketName, logData); } contents.add(listEntry); resultKeyCount++; // If max keys have been collected, set the next-marker. It might be needed for the response if the list is truncated if (resultKeyCount == maxKeys) { nextMarker = objectKey; } } } if (resultKeyCount <= maxKeys && objectInfos.size() <= maxKeys) { break; } } while (resultKeyCount <= maxKeys); reply.setContents(contents); // Prefixes are already sorted, add them to the proper data structures and populate the reply if (!commonPrefixes.isEmpty()) { ArrayList<CommonPrefixesEntry> commonPrefixesList = new ArrayList<CommonPrefixesEntry>(); for (String prefixEntry : commonPrefixes) { commonPrefixesList.add(new CommonPrefixesEntry().add(new PrefixEntry(prefixEntry))); } reply.setCommonPrefixesList(commonPrefixesList); } } else { db.rollback(); throw new AccessDeniedException("Bucket", bucketName, logData); } } else { db.rollback(); throw new NoSuchBucketException(bucketName); } db.commit(); return reply; } finally { if (db.isActive()) { db.rollback(); } } }
From source file:com.qpark.maven.plugin.securityconfig.SecurityConfigMojo.java
private String getSecurityAuthorisationSpringConfig(final XsdsUtil xsds) { List<String> annonyoums = getSplitted(this.channelPatternsAnnonymousAuthorisation); List<String> admins = getSplitted(this.channelPatternsAdminAuthorisation); List<String> create = getSplitted(this.channelPatternsCreate); List<String> read = getSplitted(this.channelPatternsRead); List<String> update = getSplitted(this.channelPatternsUpdate); List<String> delete = getSplitted(this.channelPatternsDelete); this.roleList.add("ROLE_ANONYMOUS"); this.roleList.add("ROLE_ADMIN"); this.roleList.add("ROLE_ALL_OPERATIONS"); StringBuffer sb = new StringBuffer(1024); TreeSet<String> serviceIds = new TreeSet<String>(); StringBuffer operationPolicies = new StringBuffer(1024); sb.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"); sb.append("<beans xmlns=\"http://www.springframework.org/schema/beans\"\n"); sb.append("\txmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" \n"); sb.append("\txmlns:int-security=\"http://www.springframework.org/schema/integration/security\"\n"); sb.append("\txsi:schemaLocation=\"\n"); String springVersion = this.project.getProperties().getProperty("org.springframework.version.xsd.version"); sb.append(/*from w w w. ja v a 2 s . com*/ "\t\thttp://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans"); if (springVersion != null) { sb.append("-").append(springVersion); } sb.append(".xsd\n"); sb.append( "\t\thttp://www.springframework.org/schema/integration/security http://www.springframework.org/schema/integration/security/spring-integration-security.xsd\n"); sb.append("\">\n"); sb.append(Util.getGeneratedAtXmlComment(this.getClass(), this.eipVersion)); sb.append("\n"); sb.append("\t<!-- Authorization -->\n"); sb.append("\t<!-- Role voter. -->\n"); sb.append("\t<bean id=\"eipRoleVoter\" class=\"com.qpark.eip.core.spring.security.EipRoleVoter\""); if (this.limitedAccessDataProviderBeanName != null && this.limitedAccessDataProviderBeanName.trim().length() > 0) { sb.append(">\n"); sb.append("\t\t<property name=\"eipLimitedAccessDataProvider\" ref=\""); sb.append(this.limitedAccessDataProviderBeanName); sb.append("\"/>\n"); sb.append("\t</bean>\n"); } else { sb.append("/>\n"); } sb.append( "\t<bean id=\"eipAccessDecisionManager\" class=\"com.qpark.eip.core.spring.security.EipAffirmativeBased\">\n"); sb.append("\t\t<constructor-arg>\n"); sb.append("\t\t\t<list>\n"); sb.append("\t\t\t\t<ref bean=\"eipRoleVoter\"/>\n"); sb.append("\t\t\t</list>\n"); sb.append("\t\t</constructor-arg>\n"); sb.append("\t</bean>\n"); sb.append("\n"); sb.append("\t<!-- \n"); sb.append("\tThe pattern (java.util.regexp.Pattern) of the access policies regarding \n"); sb.append("\tthe channel names used in the spring integration configurations.\n"); sb.append("\n"); sb.append("\tEach user gets the role ROLE_ANONYMOUS. If the user has an other role \n"); sb.append("\tthen ROLE_ANONYMOUS the role ROLE_COMMON need to be added too.\n"); sb.append("\n"); sb.append("\tThe user needs only one of the listed roles. All access-policy\n"); sb.append("\twill be checked until the user has a sufficient role or is\n"); sb.append("\tnot authorisized to do the operation."); sb.append("\t-->\n"); sb.append("\t<int-security:secured-channels \n"); sb.append("\t\taccess-decision-manager=\"eipAccessDecisionManager\"\n"); sb.append("\t\tauthentication-manager=\"eipAuthenticationManager\">\n"); sb.append(this.getAccessPolicyAdminAnonymous(annonyoums, "ROLE_ANONYMOUS")); sb.append(this.getAccessPolicyAdminAnonymous(admins, "ROLE_ADMIN")); sb.append(this.getAccessPolicyReadCreateUpdateDelete(read, "ROLE_READ")); sb.append(this.getAccessPolicyReadCreateUpdateDelete(create, "ROLE_CREATE")); sb.append(this.getAccessPolicyReadCreateUpdateDelete(update, "ROLE_UPDATE")); sb.append(this.getAccessPolicyReadCreateUpdateDelete(delete, "ROLE_DELETE")); sb.append("\t\t<!-- Service wide role securement -->\n"); operationPolicies.append("\t\t<!-- Operation web service operation channels -->\n"); for (ElementType element : xsds.getElementTypes()) { if (element.isRequest()) { ElementType elementResponse = XsdsUtil.findResponse(element, xsds.getElementTypes(), xsds); if (elementResponse != null) { ComplexType ctResponse = new ComplexType(elementResponse.getElement().getType(), xsds); if (ctResponse != null && !ctResponse.isSimpleType() && !ctResponse.isPrimitiveType()) { String serviceRole = new StringBuffer(32).append("ROLE_") .append(element.getServiceId().toUpperCase()).toString(); String serviceVersionLessRole = serviceRole; if (serviceRole.indexOf(".V") > 0) { serviceVersionLessRole = serviceRole.substring(0, serviceRole.indexOf(".V")); } String operationRole = new StringBuffer(64).append(serviceRole).append("_") .append(element.getOperationName().toUpperCase()).toString(); if (!serviceIds.contains(element.getServiceId())) { serviceIds.add(element.getServiceId()); this.roleList.add(serviceRole); sb.append("\t\t<int-security:access-policy pattern=\""); sb.append(element.getChannelSecurityPatternService()); sb.append("\" send-access=\""); sb.append(serviceRole); if (!serviceRole.equals(serviceVersionLessRole)) { sb.append(", "); sb.append(serviceVersionLessRole); this.roleList.add(serviceVersionLessRole); } sb.append(", ROLE_ALL_OPERATIONS"); sb.append("\" receive-access=\""); sb.append(serviceRole); if (!serviceRole.equals(serviceVersionLessRole)) { sb.append(", "); sb.append(serviceVersionLessRole); this.roleList.add(serviceVersionLessRole); } sb.append(", ROLE_ALL_OPERATIONS"); sb.append("\" />\n"); } operationPolicies.append("\t\t<int-security:access-policy pattern=\""); operationPolicies.append(element.getChannelSecurityPatternOperation()); operationPolicies.append("\" send-access=\""); operationPolicies.append(operationRole); operationPolicies.append(", "); operationPolicies.append(serviceRole); operationPolicies.append(", ROLE_ALL_OPERATIONS"); operationPolicies.append("\" receive-access=\""); operationPolicies.append(operationRole); operationPolicies.append(", "); operationPolicies.append(serviceRole); operationPolicies.append(", ROLE_ALL_OPERATIONS"); operationPolicies.append("\" />\n"); this.roleList.add(operationRole); } } } } sb.append(operationPolicies); sb.append("\t</int-security:secured-channels>\n"); sb.append("</beans>\n"); return sb.toString(); }
From source file:com.eucalyptus.walrus.WalrusFSManager.java
@Override public ListBucketResponseType listBucket(ListBucketType request) throws WalrusException { ListBucketResponseType reply = (ListBucketResponseType) request.getReply(); EntityWrapper<BucketInfo> db = EntityWrapper.get(BucketInfo.class); try {// w w w . jav a 2s. c o m String bucketName = request.getBucket(); BucketInfo bucketInfo = new BucketInfo(bucketName); bucketInfo.setHidden(false); List<BucketInfo> bucketList = db.queryEscape(bucketInfo); Context ctx = Contexts.lookup(); Account account = ctx.getAccount(); int maxKeys = -1; String maxKeysString = request.getMaxKeys(); if (maxKeysString != null) { maxKeys = Integer.parseInt(maxKeysString); if (maxKeys < 0) { throw new InvalidArgumentException("max-keys", "Argument max-keys must be an integer between 0 and " + Integer.MAX_VALUE); } } else { maxKeys = WalrusProperties.MAX_KEYS; } if (bucketList.size() > 0) { BucketInfo bucket = bucketList.get(0); BucketLogData logData = bucket.getLoggingEnabled() ? request.getLogData() : null; if (logData != null) { updateLogData(bucket, logData); reply.setLogData(logData); } if (Contexts.lookup().hasAdministrativePrivileges()) { try { if (bucketHasSnapshots(bucketName)) { db.rollback(); throw new NoSuchBucketException(bucketName); } } catch (Exception e) { db.rollback(); throw new InternalErrorException(e); } } String prefix = request.getPrefix(); String delimiter = request.getDelimiter(); String marker = request.getMarker(); reply.setName(bucketName); reply.setIsTruncated(false); reply.setPrefix(prefix); reply.setMarker(marker); reply.setDelimiter(delimiter); reply.setMaxKeys(maxKeys); if (maxKeys == 0) { // No keys requested, so just return reply.setContents(new ArrayList<ListEntry>()); reply.setCommonPrefixesList(new ArrayList<CommonPrefixesEntry>()); db.commit(); return reply; } final int queryStrideSize = maxKeys + 1; EntityWrapper<ObjectInfo> dbObject = db.recast(ObjectInfo.class); ObjectInfo searchObj = new ObjectInfo(); searchObj.setBucketName(bucketName); searchObj.setLast(true); searchObj.setDeleted(false); Criteria objCriteria = dbObject.createCriteria(ObjectInfo.class); objCriteria.add(Example.create(searchObj)); objCriteria.addOrder(Order.asc("objectKey")); objCriteria.setMaxResults(queryStrideSize); // add one to, hopefully, indicate truncation in one call if (!Strings.isNullOrEmpty(marker)) { // The result set should be exclusive of the marker. marker could be a common prefix from a previous response. Look for keys that // lexicographically follow the marker and don't contain the marker as the prefix. objCriteria.add(Restrictions.gt("objectKey", marker)); } else { marker = ""; } if (!Strings.isNullOrEmpty(prefix)) { objCriteria.add(Restrictions.like("objectKey", prefix, MatchMode.START)); } else { prefix = ""; } // Ensure not null. if (Strings.isNullOrEmpty(delimiter)) { delimiter = ""; } List<ObjectInfo> objectInfos = null; int resultKeyCount = 0; ArrayList<ListEntry> contents = new ArrayList<ListEntry>(); // contents for reply String nextMarker = null; TreeSet<String> commonPrefixes = new TreeSet<String>(); int firstResult = -1; // Iterate over result sets of size maxkeys + 1 do { // Start listing from the 0th element and increment the first element to be listed by the query size objCriteria.setFirstResult(queryStrideSize * (++firstResult)); objectInfos = (List<ObjectInfo>) objCriteria.list(); if (objectInfos.size() > 0) { for (ObjectInfo objectInfo : objectInfos) { String objectKey = objectInfo.getObjectKey(); // Check if it will get aggregated as a commonprefix if (!Strings.isNullOrEmpty(delimiter)) { String[] parts = objectKey.substring(prefix.length()).split(delimiter); if (parts.length > 1) { String prefixString = prefix + parts[0] + delimiter; if (!StringUtils.equals(prefixString, marker) && !commonPrefixes.contains(prefixString)) { if (resultKeyCount == maxKeys) { // This is a new record, so we know we're truncating if this is true reply.setNextMarker(nextMarker); reply.setIsTruncated(true); resultKeyCount++; break; } commonPrefixes.add(prefixString); resultKeyCount++; // count the unique commonprefix as a single return entry // If max keys have been collected, set the next-marker. It might be needed for the response if the list is // truncated // If the common prefixes hit the limit set by max-keys, next-marker is the last common prefix if (resultKeyCount == maxKeys) { nextMarker = prefixString; } } continue; } } if (resultKeyCount == maxKeys) { // This is a new (non-commonprefix) record, so we know we're truncating reply.setNextMarker(nextMarker); reply.setIsTruncated(true); resultKeyCount++; break; } // Process the entry as a full key listing ListEntry listEntry = new ListEntry(); listEntry.setKey(objectKey); listEntry.setEtag(objectInfo.getEtag()); listEntry.setLastModified( DateFormatter.dateToListingFormattedString(objectInfo.getLastModified())); listEntry.setStorageClass(objectInfo.getStorageClass()); listEntry.setSize(objectInfo.getSize()); listEntry.setStorageClass(objectInfo.getStorageClass()); try { Account ownerAccount = Accounts.lookupAccountById(objectInfo.getOwnerId()); listEntry.setOwner( new CanonicalUser(ownerAccount.getCanonicalId(), ownerAccount.getName())); } catch (AuthException e) { db.rollback(); throw new AccessDeniedException("Bucket", bucketName, logData); } contents.add(listEntry); resultKeyCount++; // If max keys have been collected, set the next-marker. It might be needed for the response if the list is truncated if (resultKeyCount == maxKeys) { nextMarker = objectKey; } } } if (resultKeyCount <= maxKeys && objectInfos.size() <= maxKeys) { break; } } while (resultKeyCount <= maxKeys); reply.setContents(contents); // Prefixes are already sorted, add them to the proper data structures and populate the reply if (!commonPrefixes.isEmpty()) { ArrayList<CommonPrefixesEntry> commonPrefixesList = new ArrayList<CommonPrefixesEntry>(); for (String prefixEntry : commonPrefixes) { commonPrefixesList.add(new CommonPrefixesEntry(prefixEntry)); } reply.setCommonPrefixesList(commonPrefixesList); } } else { db.rollback(); throw new NoSuchBucketException(bucketName); } db.commit(); return reply; } finally { if (db.isActive()) { db.rollback(); } } }