Java tutorial
/** * Copyright (C) 2010 University of Washington * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package org.opendatakit.common.persistence.engine.gae; import java.math.BigDecimal; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.opendatakit.common.persistence.CommonFieldsBase; import org.opendatakit.common.persistence.DataField; import org.opendatakit.common.persistence.DataField.DataType; import org.opendatakit.common.persistence.EntityKey; import org.opendatakit.common.persistence.QueryResult; import org.opendatakit.common.persistence.QueryResumePoint; import org.opendatakit.common.persistence.engine.EngineUtils; import org.opendatakit.common.persistence.exception.ODKDatastoreException; import org.opendatakit.common.persistence.exception.ODKOverQuotaException; import org.opendatakit.common.security.User; import org.opendatakit.common.utils.WebUtils; import com.google.appengine.api.datastore.DatastoreService; import com.google.appengine.api.datastore.FetchOptions; import com.google.appengine.api.datastore.PreparedQuery; import com.google.appengine.api.datastore.Query; import com.google.appengine.api.datastore.Query.CompositeFilterOperator; import com.google.appengine.api.datastore.Query.FilterOperator; import com.google.appengine.api.datastore.Query.SortDirection; import com.google.apphosting.api.ApiProxy.OverQuotaException; /** * * @author wbrunette@gmail.com * @author mitchellsundt@gmail.com * */ public class QueryImpl implements org.opendatakit.common.persistence.Query { private static final boolean isWorkingZigZagEqualityFiltering = false; static final Map<FilterOperation, FilterOperator> operationMap = new HashMap<FilterOperation, FilterOperator>(); static { operationMap.put(FilterOperation.EQUAL, FilterOperator.EQUAL); operationMap.put(FilterOperation.NOT_EQUAL, FilterOperator.NOT_EQUAL); operationMap.put(FilterOperation.GREATER_THAN, FilterOperator.GREATER_THAN); operationMap.put(FilterOperation.GREATER_THAN_OR_EQUAL, FilterOperator.GREATER_THAN_OR_EQUAL); operationMap.put(FilterOperation.LESS_THAN, FilterOperator.LESS_THAN); operationMap.put(FilterOperation.LESS_THAN_OR_EQUAL, FilterOperator.LESS_THAN_OR_EQUAL); } private final String loggingContextTag; private final CommonFieldsBase relation; private final DatastoreImpl datastore; private final User user; private final Log logger; private final ExecutionTimeLogger gaeCostLogger; private static final long ACTIVE_COST_LOGGING_CHECK_INTERVAL = 10 * 1000; // 10 // seconds static long costLoggingMinimumMegacyclesThreshold = 10 * 1200; // 10 // seconds private static long milliLastCheck = 0L; final static synchronized void updateCostLoggingThreshold(DatastoreImpl datastore) { Log logger = LogFactory.getLog(QueryImpl.class); long currentTime = System.currentTimeMillis(); if (milliLastCheck + ACTIVE_COST_LOGGING_CHECK_INTERVAL < currentTime) { milliLastCheck = currentTime;// update early in case an exception is // thrown... try { com.google.appengine.api.datastore.Query query = new Query("_COST_LOGGING_"); PreparedQuery pq = datastore.getDatastoreService().prepare(query); logger.debug("costLogging fetch."); List<com.google.appengine.api.datastore.Entity> eList = pq .asList(FetchOptions.Builder.withDefaults()); datastore.recordQueryUsage("_COST_LOGGING_", eList.size()); if (eList.isEmpty()) { costLoggingMinimumMegacyclesThreshold = 10 * 1200; // 10 seconds... logger.warn("writing 10-second cost logging threshold record"); com.google.appengine.api.datastore.Entity e = new com.google.appengine.api.datastore.Entity( "_COST_LOGGING_", "T" + WebUtils.iso8601Date(new Date())); e.setProperty("COST_LOGGING_MEGACYCLE_THRESHOLD", 10 * 1200); // 10 // seconds... e.setProperty("LAST_UPDATE_DATE", new Date()); datastore.getDatastoreService().put(e); } else { Long newValue = null; for (com.google.appengine.api.datastore.Entity e : eList) { Object o = e.getProperty("COST_LOGGING_MEGACYCLE_THRESHOLD"); if (o != null) { if (o instanceof Long) { Long l = (Long) o; if (newValue == null || newValue.compareTo(l) > 0) { newValue = l; } else { logger.warn("deleting superceded logging threshold record"); datastore.getDatastoreService().delete(e.getKey()); } } else if (o instanceof Integer) { Integer i = (Integer) o; Long l = Long.valueOf(i); if (newValue == null || newValue.compareTo(l) > 0) { newValue = l; } else { logger.warn("deleting superceded logging threshold record"); datastore.getDatastoreService().delete(e.getKey()); } } else if (o instanceof String) { String s = (String) o; try { Long l = Long.parseLong(s); if (newValue == null || newValue.compareTo(l) > 0) { newValue = l; } else { logger.warn("deleting superceded logging threshold record"); datastore.getDatastoreService().delete(e.getKey()); } } catch (NumberFormatException ex) { logger.warn("deleting superceded logging threshold record"); datastore.getDatastoreService().delete(e.getKey()); } } else { logger.warn("deleting superceded logging threshold record"); datastore.getDatastoreService().delete(e.getKey()); } } } if (newValue == null) { logger.warn("resetting cost logging to 10 second (12000 megacycle) threshold"); costLoggingMinimumMegacyclesThreshold = 10 * 1200; // 10 seconds... } else if (costLoggingMinimumMegacyclesThreshold != newValue.longValue()) { logger.warn("changing cost logging to " + newValue + " megacycle threshold"); costLoggingMinimumMegacyclesThreshold = newValue; } } } catch (Exception e) { e.printStackTrace(); logger.error("exception while updating cost logging threshold: " + e.getMessage()); } } } private final List<Tracker> filterList = new ArrayList<Tracker>(); private final List<SortTracker> sortList = new ArrayList<SortTracker>(); public QueryImpl(CommonFieldsBase relation, String loggingContextTag, DatastoreImpl datastore, User user) { this.relation = relation; this.loggingContextTag = loggingContextTag; this.datastore = datastore; this.user = user; this.logger = LogFactory.getLog(QueryImpl.class); this.gaeCostLogger = new ExecutionTimeLogger(datastore, loggingContextTag, relation); } private SimpleFilterTracker constructFilter(DataField attribute, FilterOperation op, Object value) { // do everything locally except the first one (later)... if (attribute.getDataType() == DataType.DECIMAL) { if (value != null) { // ensure the value is always a BigDecimal and always rounded to scale BigDecimal bd; if (value instanceof BigDecimal) { bd = (BigDecimal) value; } else { bd = new BigDecimal(value.toString()); } bd = bd.setScale(attribute.getNumericScale(), BigDecimal.ROUND_HALF_UP); return new SimpleFilterTracker(attribute, op, bd); } else { return new SimpleFilterTracker(attribute, op, null); } } else { return new SimpleFilterTracker(attribute, op, value); } } @Override public void addFilter(DataField attribute, FilterOperation op, Object value) { filterList.add(constructFilter(attribute, op, value)); } @Override public void addValueSetFilter(DataField attribute, Collection<?> valueSet) { // do everything locally except the first one (later)... if (attribute.getDataType() == DataType.DECIMAL) { List<BigDecimal> bdList = new ArrayList<BigDecimal>(); for (Object value : valueSet) { if (value != null) { // ensure the value is always a BigDecimal and always rounded to scale BigDecimal bd; if (value instanceof BigDecimal) { bd = (BigDecimal) value; } else { bd = new BigDecimal(value.toString()); } bd = bd.setScale(attribute.getNumericScale(), BigDecimal.ROUND_HALF_UP); bdList.add(bd); } else { bdList.add(null); } } filterList.add(new ValueSetFilterTracker(attribute, bdList)); } else { filterList.add(new ValueSetFilterTracker(attribute, valueSet)); } } @Override public void addSort(DataField attribute, Direction direction) { // do the sort locally -- later... sortList.add(new SortTracker(attribute, direction)); } /** * * @param dominantSortAttr * @param startCursorFilter * @return true if there is at least one filter criteria against the dominant sort attribute */ private boolean hasDominantAttributeFilter(DataField dominantSortAttr, SimpleFilterTracker startCursorFilter) { // apply the startCursor filter on dominant attr. if (startCursorFilter != null) { return true; } // add any other filter conditions on the dominant sort attribute. // e.g., for "between x and y" types of queries. for (Tracker t : filterList) { if (dominantSortAttr.equals(t.getAttribute())) { return true; } } return false; } /** * If there is no filter criteria on the dominant sort attribute, we need * to query the database to find the min/max value for the attribute and then * apply that to the query. This is a work-around for broken GAE functionality. * * @return null if no records in table; otherwise, produce the needed filter. * @throws ODKDatastoreException */ private SimpleFilterTracker getImpliedDominantAttributeFilter() throws ODKDatastoreException { // GAE production throws an exception when a query with equality // filters and a sort on a column not constrained by any filter // is issued. // // To work around this, issue a query to return the current min/max // value of the dominant sort attribute and apply a GE/LE // constraint using that value. This is effectively a no-op, // but keeps GAE happy. // // this is the dominant sort: SortTracker dominantSort = sortList.get(0); DataField dominantSortAttr = dominantSort.getAttribute(); SortDirection dominantSortDirection = dominantSort.direction.equals(Direction.ASCENDING) ? SortDirection.ASCENDING : SortDirection.DESCENDING; DatastoreService ds = datastore.getDatastoreService(); EntityRowMapper m = new EntityRowMapper(relation, user); Query orderingHack = new com.google.appengine.api.datastore.Query( relation.getSchemaName() + "." + relation.getTableName()); orderingHack.addSort(dominantSort.getAttribute().getName(), dominantSortDirection); PreparedQuery orderingPrep = ds.prepare(orderingHack); logger.debug("hqrLoop: finding min/max in " + relation.getSchemaName() + "." + relation.getTableName() + " of dominantSortAttr: " + dominantSortAttr.getName()); List<com.google.appengine.api.datastore.Entity> values = orderingPrep .asList(FetchOptions.Builder.withDefaults().prefetchSize(1).chunkSize(20).limit(3)); datastore.recordQueryUsage(relation, values.size()); if (values == null || values.isEmpty()) { // the table is empty -- no need to go further... return null; } Object dominantFilterValue = null; // and apply the filter... try { CommonFieldsBase odkEntity = (CommonFieldsBase) m.mapRow(datastore, values.get(0), 0); dominantFilterValue = EngineUtils.getDominantSortAttributeValue(odkEntity, dominantSortAttr); } catch (SQLException e) { throw new ODKDatastoreException("[" + loggingContextTag + "] Unable to complete request", e); } SimpleFilterTracker impliedDominantFilter = constructFilter(dominantSortAttr, dominantSort.direction.equals(Direction.ASCENDING) ? FilterOperation.GREATER_THAN_OR_EQUAL : FilterOperation.LESS_THAN_OR_EQUAL, dominantFilterValue); return impliedDominantFilter; } /** * Construct the query appropriate for this fragment of the result-set production. * * @param startCursorFilter * @param impliedDominantFilter * @return * @throws ODKOverQuotaException * @throws ODKDatastoreException */ private PreparedQuery prepareQuery(SimpleFilterTracker startCursorFilter, SimpleFilterTracker impliedDominantFilter) throws ODKOverQuotaException, ODKDatastoreException { SortTracker dominantSort = sortList.get(0); DataField dominantSortAttr = dominantSort.getAttribute(); try { Query hack = new com.google.appengine.api.datastore.Query( relation.getSchemaName() + "." + relation.getTableName()); ArrayList<com.google.appengine.api.datastore.Query.Filter> filters = new ArrayList<com.google.appengine.api.datastore.Query.Filter>(); // add all the dominant filter conditions... // apply the startCursor filter on dominant attr. if (startCursorFilter != null) { startCursorFilter.setFilter(filters); } // add any other filter conditions on the dominant sort attribute. // e.g., for "between x and y" types of queries. for (Tracker t : filterList) { if (dominantSortAttr.equals(t.getAttribute())) { t.setFilter(filters); } } if (impliedDominantFilter != null) { // and apply the implicit filter... impliedDominantFilter.setFilter(filters); } // and add all other equality filter conditions. if (isWorkingZigZagEqualityFiltering) { // GAE: this doesn't work in production, // though the ZigZag queries are supposed // to support this... for (Tracker t : filterList) { if (!dominantSortAttr.equals(t.getAttribute())) { if (t instanceof SimpleFilterTracker) { SimpleFilterTracker st = (SimpleFilterTracker) t; if (st.isEqualityTest()) { st.setFilter(filters); } } } } } if (filters.size() == 1) { hack.setFilter(filters.get(0)); } else if (filters.size() > 1) { hack.setFilter(new Query.CompositeFilter(CompositeFilterOperator.AND, filters)); } // add the dominant sort. SortDirection dominantSortDirection = dominantSort.direction.equals(Direction.ASCENDING) ? SortDirection.ASCENDING : SortDirection.DESCENDING; hack.addSort(dominantSort.getAttribute().getName(), dominantSortDirection); // subordinate sorts cannot be applied // GAE production doesn't like them. gaeCostLogger.declareQuery(hack); DatastoreService ds = datastore.getDatastoreService(); return ds.prepare(hack); } catch (OverQuotaException e) { throw new ODKOverQuotaException("[" + loggingContextTag + "] Quota exceeded", e); } catch (Exception e) { throw new ODKDatastoreException("[" + loggingContextTag + "] Unable to complete request", e); } } /** * Inner action function that can fill odkEntities with > fetchLimit+1 entries * beyond the set of entries matching the initial dominantSortAttr value. * We must gather all end values matching the dominant sort attribute's value * of the 'final' value and then grab the value after those as well. This * is because the 'final' value can have nested sorts applied that will reorder * the sequence. * * @param odkEntities * -- list of entities being assembled. * @param startCursorFilter * -- filter appropriate for this cursor. * @param fetchLimit * -- number of records the requester wants. * @return * @throws ODKDatastoreException * @throws ODKOverQuotaException */ private void chunkFetch(ResultContainer odkEntities, SimpleFilterTracker startCursorFilter, int fetchLimit) throws ODKDatastoreException, ODKOverQuotaException { // Step 1: create a prepared query that we may repeatedly // fetch values from using a chunk size, fetch limit and // fetch offset until we have fetched enough records into // the odkEntities list to ensure that we can return a // total of (fetchLimit+1) values (or exhaust the return set). // this is the dominant sort: SortTracker dominantSort = sortList.get(0); DataField dominantSortAttr = dominantSort.getAttribute(); // Test whether we have filters on the dominant sort attribute. // If we don't have filters on the dominant sort attribute, // then interrogate the database to establish an implied filter. SimpleFilterTracker impliedDominantFilter = null; if (!hasDominantAttributeFilter(dominantSortAttr, startCursorFilter)) { impliedDominantFilter = getImpliedDominantAttributeFilter(); if (impliedDominantFilter == null) { // we need a filter but there are no items in the table... return; // no data... } } EntityRowMapper m = new EntityRowMapper(relation, user); // Fetch chunks bigger that the default... // Bulk fetches are dead slow if the chunks are small. // This burns quota but makes fetches that filter data faster. int chunkSize = 2048; // We always start the first fetch with an offset of zero // even if this is a resumed query (startCursorFilter != null). // This is because we will be applying subordinate sorts. // We therefore don't know what the true value order is during // the GAE fetch phase and therefore cannot apply the last-URI // filtering or the appropriate fetch offset until after all // candidate values are returned. // // We do care about the fetchOffset when we issue subsequent // requests against the prepared query in order to flesh out // the result list. // int fetchOffset = 0; // track the current first entity of the result set. // this affects whether we re-issue the prepared query // or construct a new query with an updated startCursorFilter. CommonFieldsBase odkFirstEntityOfCurrentDominantValue = null; boolean dominantSortAttrValueHasChanged = false; int idx = 0; // for logging and debugging only... // since the subordinate sorts rearrange the data sharing the same // dominantSort attribute value, we must gather all matching start // values then all values up to the fetchLimit, then all matching // end values and then one more record to determine if there are // additional records. boolean possiblyBeforeStartCursor = (startCursorFilter != null); int sizeQuestionableFirstMatches = 0; boolean mustReadEverything = (fetchLimit == 0); WorkingValues w = new WorkingValues(); boolean hasQueryResults = true; for (; hasQueryResults;) { logger.debug("hqrLoop: " + idx + " fetchOffset: " + fetchOffset + " startCursor " + ((startCursorFilter == null) ? "<<none>>" : ((startCursorFilter.value == null) ? "null" : startCursorFilter.value.toString()))); // Since we are filtering locally, we need to grab a chunk of values // in the expectation that most will fail the filter. // // We also need to handle the timing out of the query to GAE. This // means we must artificially limit the production of results for the // query so that the query itself does not time out in GAE. // // Prefetch only 105 records to handle the common case of an // unfiltered paged display of 100 submissions. FetchOptions options = FetchOptions.Builder.withDefaults().chunkSize(chunkSize).prefetchSize(105) .offset(fetchOffset).limit(8 * chunkSize); logger.debug("hqrLoop: executing preparedQuery on " + relation.getSchemaName() + "." + relation.getTableName()); w.idx = idx; w.fetchOffset = fetchOffset; w.startCursorFilter = startCursorFilter; w.possiblyBeforeStartCursor = possiblyBeforeStartCursor; w.sizeQuestionableFirstMatches = sizeQuestionableFirstMatches; w.odkFirstEntityOfCurrentDominantValue = odkFirstEntityOfCurrentDominantValue; w.dominantSortAttrValueHasChanged = dominantSortAttrValueHasChanged; w.odkAdditionalEntities.clear(); try { { // scope iterable to this block -- can be GC'd on exit from it. Iterable<com.google.appengine.api.datastore.Entity> it; try { PreparedQuery preparedHack = prepareQuery(startCursorFilter, impliedDominantFilter); it = preparedHack.asIterable(options); } catch (OverQuotaException e) { datastore.recordQueryUsage(relation, 0); throw new ODKOverQuotaException("[" + loggingContextTag + "] Quota exceeded", e); } catch (Exception e) { datastore.recordQueryUsage(relation, 0); throw new ODKDatastoreException("[" + loggingContextTag + "] Unable to complete request", e); } // loop while the query returns at least one result... // Calls recordQueryUsage -- including during exceptions. hasQueryResults = fetchResults(it, dominantSort, dominantSortAttr, m, mustReadEverything, fetchLimit, options.getLimit(), odkEntities.size(), w); } // and if we succeeded, we update the actual state to that of the // fetched results. This loop makes extensive use of the // fetchOffset to index into the remainder of the query set // even when we re-issue the query. Note that the above // query-set will return only up to options.getLimit() values, // and there may be many more values beyond that. idx = w.idx; fetchOffset = w.fetchOffset; startCursorFilter = w.startCursorFilter; possiblyBeforeStartCursor = w.possiblyBeforeStartCursor; sizeQuestionableFirstMatches = w.sizeQuestionableFirstMatches; odkFirstEntityOfCurrentDominantValue = w.odkFirstEntityOfCurrentDominantValue; dominantSortAttrValueHasChanged = w.dominantSortAttrValueHasChanged; for (CommonFieldsBase cb : w.odkAdditionalEntities) { odkEntities.add(cb); } } catch (ODKOverQuotaException e) { throw e; } catch (ODKDatastoreException e) { // we had an exception -- this might be a cursor timeout(!) // Step down the chunkSize and try again. The original // WorkingValues needed to restart the query are unchanged, // so we can safely reissue the query. if (chunkSize > 64) { logger.warn("Retrying fetch with a smaller chunk size: " + e.getMessage()); chunkSize /= 4; } else { throw e; } } } logger.debug("hqrLoop: done fetching everything!"); } private final static class WorkingValues { int idx; int fetchOffset; SimpleFilterTracker startCursorFilter; boolean possiblyBeforeStartCursor; int sizeQuestionableFirstMatches; CommonFieldsBase odkFirstEntityOfCurrentDominantValue; boolean dominantSortAttrValueHasChanged; List<CommonFieldsBase> odkAdditionalEntities = new ArrayList<CommonFieldsBase>(); WorkingValues() { } } /** * Updates WorkingValues with current status values. * This fetches the records from the result-set iterator * and applies the filter criteria to them. It assumes * the query production will return a consistently ordered * set of results. * * @param it * @param dominantSort * @param dominantSortAttr * @param m * @param mustReadEverything * @param fetchLimit * @param odkEntitiesSize * @param w * @return false if completely done, true if still more work to be done. * @throws ODKDatastoreException * @throws ODKOverQuotaException */ private final boolean fetchResults(Iterable<com.google.appengine.api.datastore.Entity> it, SortTracker dominantSort, DataField dominantSortAttr, EntityRowMapper m, boolean mustReadEverything, int fetchLimit, Integer readSetLimit, int odkEntitiesSize, WorkingValues w) throws ODKDatastoreException, ODKOverQuotaException { // loop as long as the query returns at least one result... boolean hasQueryResults = false; try { int readSetCount = 0; try { for (com.google.appengine.api.datastore.Entity gaeEntity : it) { hasQueryResults = true; readSetCount++; CommonFieldsBase odkEntity = (CommonFieldsBase) m.mapRow(datastore, gaeEntity, w.idx++); // determine whether this odkEntity shares the same dominantSortAttr // value as the one before it. boolean matchingDominantAttr; // reset the fetch offset to zero and update the startCursorFilter // if the matchingAttr has changed. if (w.odkFirstEntityOfCurrentDominantValue == null) { // initialize counters and update reference entity w.fetchOffset = 0; w.odkFirstEntityOfCurrentDominantValue = odkEntity; // matchingDominantAttr is considered true for the first result... matchingDominantAttr = true; } else { matchingDominantAttr = EngineUtils.hasMatchingDominantSortAttribute( w.odkFirstEntityOfCurrentDominantValue, odkEntity, dominantSortAttr); if (!matchingDominantAttr) { // reset counters and update reference entity w.fetchOffset = 0; w.odkFirstEntityOfCurrentDominantValue = odkEntity; w.dominantSortAttrValueHasChanged = true; w.startCursorFilter = new SimpleFilterTracker(dominantSortAttr, dominantSort.direction.equals(Direction.ASCENDING) ? FilterOperation.GREATER_THAN_OR_EQUAL : FilterOperation.LESS_THAN_OR_EQUAL, EngineUtils.getDominantSortAttributeValue(odkEntity, dominantSortAttr)); } } w.fetchOffset++; // if we have read enough records to satisfy the fetchLimit, we // only need to continue reading records until matchingDominantAttr // becomes false. This indicates that we have collected all the // records that could possibly be rearranged by subordinate sorts. // if (!mustReadEverything && !matchingDominantAttr && !w.possiblyBeforeStartCursor && odkEntitiesSize + w.odkAdditionalEntities.size() > fetchLimit + w.sizeQuestionableFirstMatches + 1) { // we're done! return false; } // determine if this result passes all filters... boolean passed = true; for (Tracker t : filterList) { if (!t.passFilter(odkEntity)) { passed = false; break; } } if (passed) { if (w.possiblyBeforeStartCursor && w.dominantSortAttrValueHasChanged) { // We are starting to process the result set. // // Until the dominantSortAttr value changes from the // value initially given in the constraint, we cannot // be sure that we have fetched any values that will // eventually be returned to the caller. Subordinate // sorts will rearrange the those values and until we // have the final sorted ordering, we cannot exclude // values before the last previously-returned URI. // // So we must accumulate all of those values and not // count them toward the fetch limit. Once we do see // a change in the dominantSortAttr value, we know // we will have at least one value in the result set. // // Remember the number of questionable elements... w.possiblyBeforeStartCursor = false; w.sizeQuestionableFirstMatches = odkEntitiesSize + w.odkAdditionalEntities.size(); } // add the match... w.odkAdditionalEntities.add(odkEntity); } } } finally { datastore.recordQueryUsage(relation, readSetCount); } if (readSetLimit != null && readSetCount < readSetLimit) { // we read all the records that were available on the server // and didn't hit our batch fetch limit... we can stop now. return false; } logger.debug("hqrLoop: done with one batch fetch!"); } catch (SQLException e) { e.printStackTrace(); throw new ODKDatastoreException("[" + loggingContextTag + "] SQL: " + gaeCostLogger.queryString + " exception: " + e.getMessage(), e); } catch (OverQuotaException e) { throw new ODKOverQuotaException("[" + loggingContextTag + "] Quota exceeded", e); } catch (Exception e) { e.printStackTrace(); throw new ODKDatastoreException("[" + loggingContextTag + "] SQL: " + gaeCostLogger.queryString + " exception: " + e.getMessage(), e); } return hasQueryResults; } /** * If the primary key does not already have a sort order applied, * ensure that it is ordered in the same ascending/descending * order as the dominant sort parameter. * * @param dominantSort */ private void enforcePrimaryKeyOrdering(SortTracker dominantSort) { // if we don't have any sort on the PK, add one // direction of PK sort matches that of dominant sort boolean isUriSortAlreadyPresent = false; for (SortTracker st : sortList) { if (st.attribute.equals(relation.primaryKey)) { isUriSortAlreadyPresent = true; break; } } if (!isUriSortAlreadyPresent) { // direction of PK sort matches that of dominant sort // // NOTE: if a PK sort is already defined, it is up to the // caller to alter its sense in a new query when using // a resume point to fetch the records preceding // that resume point. addSort(relation.primaryKey, dominantSort.direction); } } /** * Uses chunkFetch(...) to fetch at least fetchLimit+1 values beyond * the start-value (or all values, if fetchLimit is zero). Then orders * the returned values and filters them w.r.t. the start cursor's PK * restriction, constructing and returning the ordered list of results * to the caller. * * @param startCursor * @param fetchLimit * @return * @throws ODKDatastoreException * @throws ODKOverQuotaException */ private CoreResult coreExecuteQuery(QueryResumePoint startCursor, int fetchLimit) throws ODKDatastoreException, ODKOverQuotaException { // get the dominant sort definition if (sortList.isEmpty()) { throw new IllegalStateException("expected at least one sort criteria"); } // this is the dominant sort: SortTracker dominantSort = sortList.get(0); DataField dominantSortAttr = dominantSort.getAttribute(); enforcePrimaryKeyOrdering(dominantSort); // we allow the same query to be executed multiple times with different // start cursors (resume points). To do that, we maintain the additional // start cursor filter separately from the caller-specified list of // query filters. SimpleFilterTracker startCursorFilter = null; if (startCursor != null) { DataField matchingStartCursorAttr = null; for (DataField d : relation.getFieldList()) { if (d.getName().equals(startCursor.getAttributeName())) { matchingStartCursorAttr = d; break; } } if (matchingStartCursorAttr == null) { throw new IllegalStateException("unable to find the matching attribute name " + "for dominant sort attribute in start cursor: " + startCursor.getAttributeName()); } if (!matchingStartCursorAttr.equals(dominantSortAttr)) { // the dominant sort column is different // -- the start cursor is not appropriate for this query. throw new IllegalStateException("start cursor is inappropriate for query"); } Object value = EngineUtils.getDominantSortAttributeValueFromString(startCursor.getValue(), dominantSortAttr); startCursorFilter = new SimpleFilterTracker(dominantSortAttr, dominantSort.direction.equals(Direction.ASCENDING) ? FilterOperation.GREATER_THAN_OR_EQUAL : FilterOperation.LESS_THAN_OR_EQUAL, value); } CoreResultContainer odkContainer = new CoreResultContainer(); chunkFetch(odkContainer, startCursorFilter, fetchLimit); List<CommonFieldsBase> odkEntities = odkContainer.getEntities(); // OK. We have our list of results. Now sort it... // Stable sorts nest backwards, so we do this by // applying the sorts in reverse order from their definitions. // // NOTE: since executions may be called repeatedly on the same // query, we do not want to in-place reverse or otherwise // alter the sortList. for (int i = sortList.size() - 1; i >= 0; --i) { Collections.sort(odkEntities, sortList.get(i)); } // process the list... boolean hasMoreResults = false; List<CommonFieldsBase> finalEntities = new ArrayList<CommonFieldsBase>(); String startUri = (startCursor == null) ? null : startCursor.getUriLastReturnedValue(); boolean beforeUri = (startUri != null); for (CommonFieldsBase cb : odkEntities) { if (beforeUri) { if (startUri.equals(cb.getUri())) { beforeUri = false; } } else if (fetchLimit == 0 || finalEntities.size() < fetchLimit) { finalEntities.add(cb); } else { hasMoreResults = true; break; } } return new CoreResult(finalEntities, hasMoreResults); } @Override public QueryResult executeQuery(QueryResumePoint startCursor, int fetchLimit) throws ODKDatastoreException, ODKOverQuotaException { try { CoreResult r = coreExecuteQuery(startCursor, fetchLimit); // quick exit -- empty set if (r.results.isEmpty()) { return new QueryResult(startCursor, r.results, null, startCursor, false); } // otherwise, we need to get the values of the dominantAttr and uri of the // last field. DataField dominantSortAttr = sortList.get(0).getAttribute(); // otherwise, we need to get the values of the dominantAttr and uri of the // last field. CommonFieldsBase cb; String value; // determine the resume cursor... cb = r.results.get(r.results.size() - 1); value = EngineUtils.getDominantSortAttributeValueAsString(cb, dominantSortAttr); QueryResumePoint resumeCursor = new QueryResumePoint(dominantSortAttr.getName(), value, cb.getUri(), ((startCursor != null) ? startCursor.isForwardCursor() : true)); // determine the backward cursor... cb = r.results.get(0); value = EngineUtils.getDominantSortAttributeValueAsString(cb, dominantSortAttr); QueryResumePoint backwardCursor = new QueryResumePoint(dominantSortAttr.getName(), value, cb.getUri(), !((startCursor != null) ? startCursor.isForwardCursor() : true)); return new QueryResult(startCursor, r.results, backwardCursor, resumeCursor, r.hasMoreResults); } finally { gaeCostLogger.wrapUp(); } } /** * Incoming queries that lack an sort criteria will have the * field of the first filter criteria passed down as a sort * criteria. If zig-zag queries are supported, the sort * field will be the first non-equality test in the initial * query (or the PK if there is none). */ private void establishDominantSort() { // Ensure at least one dominant sort is applied to the result set. // This allows the methods that return all matches to leverage the // core query execution logic. if (sortList.isEmpty()) { if (filterList.isEmpty()) { // use primary key, as we know that is never null. addSort(relation.primaryKey, Direction.ASCENDING); } else { // we want to sort by whatever the first filter // criteria is. The callers of the fetch-all // methods should be applying the filters in an // order that maximally excludes records so we // want to pass the first filter down to the GAE // layer. Get that to happen by sorting along that // filter dimension (which will pass down to GAE the // sort directive and the filters for that column). for (Tracker t : filterList) { if (t instanceof SimpleFilterTracker) { SimpleFilterTracker st = (SimpleFilterTracker) t; if (isWorkingZigZagEqualityFiltering) { if (st.isEqualityTest()) { continue; // zig-zag allows multiple equality tests... } } addSort(st.getAttribute(), Direction.ASCENDING); break; } } if (sortList.isEmpty()) { addSort(relation.primaryKey, Direction.ASCENDING); } } } } @Override public List<? extends CommonFieldsBase> executeQuery() throws ODKDatastoreException, ODKOverQuotaException { try { establishDominantSort(); CoreResult result = coreExecuteQuery(null, 0); return result.results; } finally { gaeCostLogger.wrapUp(); } } @Override public Set<EntityKey> executeForeignKeyQuery(CommonFieldsBase topLevelTable, DataField topLevelAuri) throws ODKDatastoreException, ODKOverQuotaException { try { List<?> keys = doExecuteDistinctValueForDataField(topLevelAuri); Set<EntityKey> keySet = new HashSet<EntityKey>(); for (Object o : keys) { String key = (String) o; // we don't have the top level records themselves. Construct the entity // keys // from the supplied relation and the value of the AURI fields in the // records // we do have. keySet.add(new EntityKey(topLevelTable, key)); } return keySet; } finally { gaeCostLogger.wrapUp(); } } @Override public List<?> executeDistinctValueForDataField(DataField dataField) throws ODKDatastoreException, ODKOverQuotaException { try { return doExecuteDistinctValueForDataField(dataField); } finally { gaeCostLogger.wrapUp(); } } private List<?> doExecuteDistinctValueForDataField(DataField dataField) throws ODKDatastoreException, ODKOverQuotaException { establishDominantSort(); SortTracker dominantSort = sortList.get(0); enforcePrimaryKeyOrdering(dominantSort); DistinctResultContainer uniqueResultContainer = new DistinctResultContainer(dataField); chunkFetch(uniqueResultContainer, null, 0); // OK. We have our list of results. Now sort it... // Stable sorts nest backwards, so we do this by // applying the sorts in reverse order from their definitions. List<Object> values = new ArrayList<Object>(uniqueResultContainer.getValueSet()); // and apply the sorting, if any, appropriate to this dataField for (int i = sortList.size() - 1; i >= 0; --i) { SortTracker t = sortList.get(i); if (t.getAttribute().equals(dataField)) { Collections.sort(values, t.getComparator()); } } return values; } }