Java tutorial
/****************************************************************************** * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. * * Software distributed under the License is distributed on an "AS IS" basis, * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for * the specific language governing rights and limitations under the License. * * The Original Code is: Jsoda * The Initial Developer of the Original Code is: William Wong (williamw520@gmail.com) * Portions created by William Wong are Copyright (C) 2012 William Wong, All Rights Reserved. * ******************************************************************************/ package wwutil.jsoda; import java.io.*; import java.net.*; import java.util.*; import java.lang.reflect.*; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.commons.beanutils.ConvertUtils; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.services.dynamodb.AmazonDynamoDBClient; import com.amazonaws.services.dynamodb.model.CreateTableRequest; import com.amazonaws.services.dynamodb.model.DeleteTableRequest; import com.amazonaws.services.dynamodb.model.ListTablesResult; import com.amazonaws.services.dynamodb.model.KeySchema; import com.amazonaws.services.dynamodb.model.KeySchemaElement; import com.amazonaws.services.dynamodb.model.ProvisionedThroughput; import com.amazonaws.services.dynamodb.model.Key; import com.amazonaws.services.dynamodb.model.PutItemRequest; import com.amazonaws.services.dynamodb.model.AttributeValue; import com.amazonaws.services.dynamodb.model.ExpectedAttributeValue; import com.amazonaws.services.dynamodb.model.GetItemRequest; import com.amazonaws.services.dynamodb.model.GetItemResult; import com.amazonaws.services.dynamodb.model.DeleteItemRequest; import com.amazonaws.services.dynamodb.model.ComparisonOperator; import com.amazonaws.services.dynamodb.model.QueryRequest; import com.amazonaws.services.dynamodb.model.QueryResult; import com.amazonaws.services.dynamodb.model.ScanRequest; import com.amazonaws.services.dynamodb.model.ScanResult; import com.amazonaws.services.dynamodb.model.Condition; import wwutil.sys.ReflectUtil; import wwutil.model.MemCacheable; import wwutil.model.annotation.DbType; import wwutil.model.annotation.Model; import wwutil.model.annotation.CachePolicy; import wwutil.model.annotation.DefaultGUID; import wwutil.model.annotation.DefaultComposite; import wwutil.model.annotation.CacheByField; /** * DynamoDB specific functions */ class DynamoDBService implements DbService { private static Log log = LogFactory.getLog(DynamoDBService.class); static final Map<String, ComparisonOperator> sOperatorMap = new HashMap<String, ComparisonOperator>() { { put(Filter.NULL, ComparisonOperator.NULL); put(Filter.NOT_NULL, ComparisonOperator.NOT_NULL); put(Filter.EQ, ComparisonOperator.EQ); put(Filter.NE, ComparisonOperator.NE); put(Filter.LE, ComparisonOperator.LE); put(Filter.LT, ComparisonOperator.LT); put(Filter.GE, ComparisonOperator.GE); put(Filter.GT, ComparisonOperator.GT); put(Filter.CONTAINS, ComparisonOperator.CONTAINS); put(Filter.NOT_CONTAINS, ComparisonOperator.NOT_CONTAINS); put(Filter.BEGINS_WITH, ComparisonOperator.BEGINS_WITH); put(Filter.BETWEEN, ComparisonOperator.BETWEEN); put(Filter.IN, ComparisonOperator.IN); } }; private Jsoda jsoda; private AmazonDynamoDBClient ddbClient; private String endPoint; // AWS Access Key ID and Secret Access Key public DynamoDBService(Jsoda jsoda, AWSCredentials cred) { this.jsoda = jsoda; this.ddbClient = new AmazonDynamoDBClient(cred); } public void shutdown() { ddbClient.shutdown(); } public DbType getDbType() { return DbType.DynamoDB; } public String getDbTypeId() { return "DYN"; } public void setDbEndpoint(String endpoint) { this.endPoint = endpoint; ddbClient.setEndpoint(endpoint); } public String getDbEndpoint() { return this.endPoint; } // Delegated Dynamodb API public void createModelTable(String modelName) { Class modelClass = jsoda.getModelClass(modelName); String table = jsoda.getModelTable(modelName); Field idField = jsoda.getIdField(modelName); String idName = getFieldAttrName(modelName, idField.getName()); Field rangeField = jsoda.getRangeField(modelName); KeySchema key = new KeySchema(); Long readTP = ReflectUtil.getAnnotationValueEx(modelClass, Model.class, "readThroughput", Long.class, new Long(10)); Long writeTP = ReflectUtil.getAnnotationValueEx(modelClass, Model.class, "writeThroughput", Long.class, new Long(5)); key.setHashKeyElement(makeKeySchemaElement(idField)); if (rangeField != null) key.setRangeKeyElement(makeKeySchemaElement(rangeField)); ddbClient.createTable(new CreateTableRequest(table, key).withProvisionedThroughput( new ProvisionedThroughput().withReadCapacityUnits(readTP).withWriteCapacityUnits(writeTP))); } private KeySchemaElement makeKeySchemaElement(Field field) { KeySchemaElement elem = new KeySchemaElement(); String attrType; if (isN(field.getType())) attrType = "N"; else attrType = "S"; // everything else has string attribute type. return elem.withAttributeName(field.getName()).withAttributeType(attrType); } public void deleteTable(String tableName) { ddbClient.deleteTable(new DeleteTableRequest(tableName)); } public List<String> listTables() { ListTablesResult list = ddbClient.listTables(); return list.getTableNames(); } public <T> void putObj(Class<T> modelClass, T dataObj, String expectedField, Object expectedValue, boolean expectedExists) throws Exception { String modelName = jsoda.getModelName(modelClass); String table = jsoda.getModelTable(modelName); PutItemRequest req = new PutItemRequest(table, objToAttrs(dataObj, modelName)); if (expectedField != null) req.setExpected(makeExpectedMap(modelName, expectedField, expectedValue, expectedExists)); ddbClient.putItem(req); } public <T> void putObjs(Class<T> modelClass, List<T> dataObjs) throws Exception { // Dynamodb has no batch put support. Emulate it. for (T obj : dataObjs) putObj(modelClass, obj, null, null, false); } public <T> T getObj(Class<T> modelClass, Object id, Object rangeKey) throws Exception { String modelName = jsoda.getModelName(modelClass); String table = jsoda.getModelTable(modelName); GetItemRequest req = new GetItemRequest(table, makeKey(modelName, id, rangeKey)); GetItemResult result = ddbClient.getItem(req); if (result.getItem() == null || result.getItem().size() == 0) return null; // not existed. return itemToObj(modelClass, result.getItem()); } public void delete(String modelName, Object id, Object rangeKey) throws Exception { String table = jsoda.getModelTable(modelName); ddbClient.deleteItem(new DeleteItemRequest(table, makeKey(modelName, id, rangeKey))); } public void batchDelete(String modelName, List idList, List rangeKeyList) throws Exception { for (int i = 0; i < idList.size(); i++) { delete(modelName, idList.get(i), rangeKeyList == null ? null : rangeKeyList.get(i)); } } public void validateFilterOperator(String operator) { if (sOperatorMap.get(operator) == null) throw new UnsupportedOperationException("Unsupported operator: " + operator); } // /** Get by a field beside the id */ // public <T> T findBy(Class<T> modelClass, String field, Object fieldValue) // throws Exception // { // String modelName = getModelName(modelClass); // T obj = (T)cacheGet(modelName, field, fieldValue); // if (obj != null) // return obj; // List<T> items = query(modelClass).filter(field, "=", fieldValue).run(); // // runQuery() has already cached the object. No need to cache it here. // return items.size() == 0 ? null : items.get(0); // } @SuppressWarnings("unchecked") public <T> long queryCount(Class<T> modelClass, Query<T> query) throws JsodaException { String modelName = jsoda.getModelName(modelClass); QueryRequest queryReq = new QueryRequest(); ScanRequest scanReq = new ScanRequest(); try { if (toRequest(query, queryReq, scanReq)) { return ddbClient.query(queryReq).getCount().intValue(); } else { return ddbClient.scan(scanReq).getCount().intValue(); } } catch (Exception e) { throw new JsodaException("Query failed. Error: " + e.getMessage(), e); } } @SuppressWarnings("unchecked") public <T> List<T> queryRun(Class<T> modelClass, Query<T> query, boolean continueFromLastRun) throws JsodaException { List<T> resultObjs = new ArrayList<T>(); if (continueFromLastRun && !queryHasNext(query)) return resultObjs; QueryRequest queryReq = new QueryRequest(); ScanRequest scanReq = new ScanRequest(); List<Map<String, AttributeValue>> items; try { if (toRequest(query, queryReq, scanReq)) { if (continueFromLastRun) queryReq.setExclusiveStartKey((Key) query.nextKey); QueryResult result = ddbClient.query(queryReq); query.nextKey = result.getLastEvaluatedKey(); items = result.getItems(); } else { if (continueFromLastRun) queryReq.setExclusiveStartKey((Key) query.nextKey); ScanResult result = ddbClient.scan(scanReq); query.nextKey = result.getLastEvaluatedKey(); items = result.getItems(); } for (Map<String, AttributeValue> item : items) { T obj = itemToObj(modelClass, item); resultObjs.add(obj); } return resultObjs; } catch (Exception e) { throw new JsodaException("Query failed. Error: " + e.getMessage(), e); } } public <T> boolean queryHasNext(Query<T> query) { return query.nextKey != null; } public String getFieldAttrName(String modelName, String fieldName) { String attrName = jsoda.getFieldAttrMap(modelName).get(fieldName); return attrName; } private boolean isN(Class valueType) { if (valueType == Integer.class || valueType == int.class) return true; if (valueType == Long.class || valueType == long.class) return true; if (valueType == Float.class || valueType == float.class) return true; if (valueType == Double.class || valueType == double.class) return true; return false; } private boolean isMultiValuetype(Class valueType) { if (valueType != null && (valueType == Integer.class || valueType == Long.class || valueType == Float.class || valueType == Double.class || valueType == String.class)) return true; return false; } private AttributeValue valueToAttr(Field field, Object value) { // Don't set the AttributeValue for null value if (value == null) return null; // Handle Set<String>, Set<Long>, or Set<Integer> field. if (Set.class.isAssignableFrom(field.getType())) { Class paramType = ReflectUtil.getGenericParamType1(field.getGenericType()); if (isMultiValuetype(paramType)) { if (isN(paramType)) { return new AttributeValue().withNS(DataUtil.toStringSet((Set) value, paramType)); } else { return new AttributeValue().withSS(DataUtil.toStringSet((Set) value, paramType)); } } } // Handle number types if (isN(field.getType())) { return new AttributeValue().withN(value.toString()); } // Delegate to DataUtil to encode the rest. return new AttributeValue().withS(DataUtil.encodeValueToAttrStr(value, field.getType())); } private Object attrToValue(Field field, AttributeValue attr) throws Exception { // Handle Set<String>, Set<Long>, or Set<Integer> field. if (Set.class.isAssignableFrom(field.getType())) { Class paramType = ReflectUtil.getGenericParamType1(field.getGenericType()); if (isMultiValuetype(paramType)) { if (isN(paramType)) return DataUtil.toObjectSet(attr.getNS(), paramType); else return DataUtil.toObjectSet(attr.getSS(), paramType); } } // Handle number types if (isN(field.getType())) { return ConvertUtils.convert(attr.getN(), field.getType()); } // Delegate to DataUtil to decode the rest. return DataUtil.decodeAttrStrToValue(attr.getS(), field.getType()); } private Map<String, AttributeValue> objToAttrs(Object dataObj, String modelName) throws Exception { Map<String, AttributeValue> attrs = new HashMap<String, AttributeValue>(); for (Map.Entry<String, String> fieldAttr : jsoda.getFieldAttrMap(modelName).entrySet()) { String fieldName = fieldAttr.getKey(); String attrName = fieldAttr.getValue(); Field field = jsoda.getField(modelName, fieldName); Object fieldValue = field.get(dataObj); AttributeValue attr = valueToAttr(field, fieldValue); if (attr != null) attrs.put(attrName, attr); // Skip setting attribute if it's null. } return attrs; } private Key makeKey(String modelName, Object id, Object rangeKey) throws Exception { if (id == null) throw new IllegalArgumentException("Id cannot be null."); Field idField = jsoda.getIdField(modelName); Field rangeField = jsoda.getRangeField(modelName); if (rangeField == null) return new Key(valueToAttr(idField, id)); else { if (rangeKey == null) throw new IllegalArgumentException( "Missing range key for the composite primary key (id,rangekey) of " + modelName); return new Key(valueToAttr(idField, id), valueToAttr(rangeField, rangeKey)); } } private Map<String, ExpectedAttributeValue> makeExpectedMap(String modelName, String expectedField, Object expectedValue, boolean expectedExists) throws Exception { if (expectedValue == null) throw new IllegalArgumentException("ExpectedValue cannot be null."); String attrName = jsoda.getFieldAttrMap(modelName).get(expectedField); Field field = jsoda.getField(modelName, expectedField); ExpectedAttributeValue cond; if (expectedExists) { cond = new ExpectedAttributeValue(expectedExists).withValue(valueToAttr(field, expectedValue)); } else { cond = new ExpectedAttributeValue(expectedExists); } Map<String, ExpectedAttributeValue> expectedMap = new HashMap<String, ExpectedAttributeValue>(); expectedMap.put(attrName, cond); return expectedMap; } private <T> T itemToObj(Class<T> modelClass, Map<String, AttributeValue> attrs) throws Exception { String modelName = jsoda.getModelName(modelClass); T dataObj = modelClass.newInstance(); // Set the attr field for (String attrName : attrs.keySet()) { Field field = jsoda.getFieldByAttr(modelName, attrName); if (field == null) { //throw new Exception("Attribute " + attrName + " from db has no corresponding field in object " + modelClass); log.warn("Attribute " + attrName + " from db has no corresponding field in model class " + modelClass); continue; } AttributeValue attr = attrs.get(attrName); Object fieldValue = attrToValue(field, attr); //log.debug("attrName " + attrName + " attr: " + attr); field.set(dataObj, fieldValue); } return dataObj; } private <T> boolean toRequest(Query<T> query, QueryRequest queryReq, ScanRequest scanReq) { addSelect(query, queryReq, scanReq); addFrom(query, queryReq, scanReq); boolean doQuery = addFilter(query, queryReq, scanReq); addOrderby(query, queryReq, scanReq, doQuery); addLimit(query, queryReq, scanReq, doQuery); queryReq.setConsistentRead(query.consistentRead); return doQuery; } private <T> void addSelect(Query<T> query, QueryRequest queryReq, ScanRequest scanReq) { if (query.selectTerms.size() == 0) return; boolean hasId = false; boolean hasRangeKey = false; Collection<String> attributesToGet = new ArrayList<String>(); for (String fieldName : query.selectTerms) { attributesToGet.add(getFieldAttrName(query.modelName, fieldName)); if (jsoda.isIdField(query.modelName, fieldName)) hasId = true; if (jsoda.isRangeField(query.modelName, fieldName)) hasRangeKey = true; } queryReq.setAttributesToGet(attributesToGet); scanReq.setAttributesToGet(attributesToGet); } private <T> void addFrom(Query<T> query, QueryRequest queryReq, ScanRequest scanReq) { queryReq.setTableName(jsoda.getModelTable(query.modelName)); scanReq.setTableName(jsoda.getModelTable(query.modelName)); } private <T> boolean addFilter(Query<T> query, QueryRequest queryReq, ScanRequest scanReq) { boolean hasIdEq = false; boolean hasRange = false; boolean doQuery; // Find out if query has an Id EQ filter and a Range filter. for (Filter filter : query.filters) { if (jsoda.isIdField(query.modelName, filter.fieldName)) { if (filter.operator.equals(Filter.EQ)) hasIdEq = true; } else if (jsoda.isRangeField(query.modelName, filter.fieldName)) { hasRange = true; } } doQuery = (hasIdEq && hasRange); if (doQuery) { log.info("Query results in a DynamoDB query."); addQueryFilter(query, queryReq); } else { log.info("Query results in a DynamoDB scan."); addScanFilter(query, scanReq); } return doQuery; } private <T> void addQueryFilter(Query<T> query, QueryRequest queryReq) { AttributeValue hashKeyValue = null; Condition rangeCondition = null; for (Filter filter : query.filters) { if (jsoda.isIdField(query.modelName, filter.fieldName)) { if (filter.operator.equals(Filter.EQ)) if (filter.operand == null) throw new IllegalArgumentException("Operand of EQ cannot be null."); else hashKeyValue = valueToAttr(jsoda.getIdField(query.modelName), filter.operand); else throw new IllegalArgumentException( "Only EQ condition is allowed on the Id field for DynamoDB."); } else if (jsoda.isRangeField(query.modelName, filter.fieldName)) { rangeCondition = toCondition(filter); } else { throw new IllegalArgumentException("Condition on " + filter.fieldName + " is not supported. DynamoDB only supports condition on the Id field and the RangeKey field."); } } if (hashKeyValue != null) queryReq.setHashKeyValue(hashKeyValue); else throw new IllegalArgumentException( "Missing an EQ condition for the Id field. DynamoDB query requires the HashKey value from the Id field."); if (rangeCondition != null) queryReq.setRangeKeyCondition(rangeCondition); } private <T> void addScanFilter(Query<T> query, ScanRequest scanReq) { Map<String, Condition> conditions = new HashMap<String, Condition>(); for (Filter filter : query.filters) { String attrName = jsoda.getFieldAttrMap(query.modelName).get(filter.fieldName); conditions.put(attrName, toCondition(filter)); } scanReq.setScanFilter(conditions); } private Condition toCondition(Filter filter) { if (Filter.BINARY_OPERATORS.contains(filter.operator)) { if (filter.operand == null) throw new IllegalArgumentException("Operand of a condition cannot be null."); return new Condition().withComparisonOperator(sOperatorMap.get(filter.operator)) .withAttributeValueList(valueToAttr(filter.field, filter.operand)); } if (Filter.UNARY_OPERATORS.contains(filter.operator)) { return new Condition().withComparisonOperator(sOperatorMap.get(filter.operator)); } if (Filter.TRINARY_OPERATORS.contains(filter.operator)) { if (filter.operand == null || filter.operand2 == null) throw new IllegalArgumentException("Operand of a condition cannot be null."); return new Condition().withComparisonOperator(sOperatorMap.get(filter.operator)).withAttributeValueList( valueToAttr(filter.field, filter.operand), valueToAttr(filter.field, filter.operand2)); } if (Filter.LIST_OPERATORS.contains(filter.operator)) { Condition cond = new Condition().withComparisonOperator(sOperatorMap.get(filter.operator)); List<AttributeValue> attrs = new ArrayList<AttributeValue>(); for (Object valueObj : filter.operands) { attrs.add(valueToAttr(filter.field, valueObj)); } cond.setAttributeValueList(attrs); return cond; } throw new UnsupportedOperationException("Condition operator " + filter.operator + " is not supported."); } private <T> void addOrderby(Query<T> query, QueryRequest queryReq, ScanRequest scanReq, boolean doQuery) { for (String orderby : query.orderbyFields) { String fieldName = orderby.substring(1); boolean forward = orderby.charAt(0) == '+'; if (!jsoda.isRangeField(query.modelName, fieldName)) throw new IllegalArgumentException("Field " + fieldName + " is not the Range Key field. DynamoDB only supports order by on the Range Key field."); if (doQuery) queryReq.setScanIndexForward(forward); else throw new IllegalArgumentException( "DynamoDB doesn't support order by on scanning. Use Id and Range Key conditions to form a query for order by."); } } private <T> void addLimit(Query<T> query, QueryRequest queryReq, ScanRequest scanReq, boolean doQuery) { if (query.limit > 0) { if (doQuery) queryReq.setLimit(query.limit); else scanReq.setLimit(query.limit); } } }