Java tutorial
/* * Copyright (c) 2009-2010 Lockheed Martin Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.hadoopvietnam.cache.memcached; import com.hadoopvietnam.cache.Cache; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import net.spy.memcached.CASMutation; import net.spy.memcached.CASMutator; import net.spy.memcached.MemcachedClient; import net.spy.memcached.transcoders.SerializingTranscoder; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; /** * Cache implementation that stores data using memcached. */ public class MemcachedCache implements Cache { /** * Instance of the logger. */ private final Log log = LogFactory.getLog(MemcachedCache.class); /** * The memcached client. */ private MemcachedClient client; /** * Maximum key expiration time allowed by memcached. */ private static final int MAX_EXPIRATION_TIME = 60 * 60 * 24 * 30; /** * Maximum number of items to keep in any memcached list. */ private int maxListSize; /** * Constructor. * * @param inClient the memcached client to set. * @param inMaxListSize the maximum size a list can grow to before it is * trimmed */ public MemcachedCache(final MemcachedClient inClient, final int inMaxListSize) { client = inClient; maxListSize = inMaxListSize; } /** * @return the maxListSize */ public int getMaxListSize() { return maxListSize; } /** * @param inMaxListSize the maxListSize to set */ public void setMaxListSize(final int inMaxListSize) { this.maxListSize = inMaxListSize; } /** * {@inheritDoc} */ public void clear() { log.trace("Flushing client"); client.flush(); } /** * {@inheritDoc} */ public Object get(final String inKey) { if (log.isTraceEnabled()) { log.trace("Getting " + inKey); } return client.get(inKey); } /** * {@inheritDoc} */ public void set(final String inKey, final Object inValue) { if (log.isTraceEnabled()) { log.trace("Setting to cache '" + inKey + "' - " + inValue); } if (inValue != null) { client.set(inKey, MAX_EXPIRATION_TIME, inValue); } else { // a null really should be a delete of the key log.warn("null passed in as parameter for key " + inKey + ". Deleting key from memcached to force client reload."); this.delete(inKey); } } /** * {@inheritDoc} */ public void set(final String inKey, final int timeExpiration, final Object inValue) { if (log.isTraceEnabled()) { log.trace("Setting to cache '" + inKey + "' - " + inValue); } if (inValue != null) { client.set(inKey, timeExpiration, inValue); } else { // a null really should be a delete of the key log.warn("null passed in as parameter for key " + inKey + ". Deleting key from memcached to force client reload."); this.delete(inKey); } } /** * {@inheritDoc} */ public ArrayList<Long> setListCAS(final String inKey, final List<Long> inValue) { // get the collection in the right shape // to assign to 'final' newInValue List<Long> tempInValue = inValue; if (tempInValue == null) { tempInValue = new ArrayList<Long>(); } // use newInValue moving forward final List<Long> newInValue = tempInValue; if (log.isTraceEnabled()) { log.trace("Comparing and swapping list '" + inKey + "', value: " + newInValue); } final ArrayList<Long> toReturn = new ArrayList<Long>(); CASMutation<Object> mutation = new CASMutation<Object>() { // This is only invoked when the list already exists. public Object getNewValue(final Object current) { Object returnToMemcache = null; // store the current value in memcache (to return to caller) // but return to memcache the return from getBytesFromList since that's // really what the caller wants in memcache try { ArrayList<Long> list = getListFromBytes(current); if (list != null) { toReturn.addAll(list); } } catch (IOException e) { log.error("Error in getListFromBytes setting memcached list with passed in value for key " + inKey + ". Exception : " + e.toString()); } try { // grab the byte[] from what the user passed in to assign to memcache returnToMemcache = getBytesFromList(newInValue); } catch (IOException e) { log.error("Error in getBytesFromList setting memcached list with passed in value for key " + inKey + ". Exception : " + e.toString()); } return returnToMemcache; } }; SerializingTranscoder transcoder = new SerializingTranscoder(); CASMutator<Object> mutator = new CASMutator<Object>(client, transcoder); // This returns whatever value was successfully stored within the cache, // either the initial list (as bytes) or a mutated existing one try { // The initial value -- only used when there's no list stored under the // key. byte[] initialValue = getBytesFromList(inValue); ArrayList<Long> list = getListFromBytes( mutator.cas(inKey, initialValue, MAX_EXPIRATION_TIME, mutation)); return toReturn; } catch (Exception e) { // CAS timeout exceeded, return null because nothing was // stored in cache. log.error("Error in setListCAS communicating with memcache. Key " + inKey + " Exception " + e.toString()); return null; } } /** * {@inheritDoc} */ public void setList(final String inKey, final List<Long> inValue) { if (log.isTraceEnabled()) { log.trace("Setting List<Long> (serialized as byte[]) to cache '" + inKey + "' - " + inValue); } if (inValue == null) { // cannot pass null to memcached log.warn("In setList, attempting to pass in NULL to memcached for key " + inKey); return; } try { byte[] bytesToSet = getBytesFromList(inValue); if (bytesToSet != null) { this.set(inKey, bytesToSet); } else { // a null here really means delete the key this.delete(inKey); } } catch (IOException e) { log.error("Error setting memcached list with passed in value for key " + inKey + ". Exception : " + e.toString()); } } /** * {@inheritDoc} */ public void delete(final String inKey) { if (log.isTraceEnabled()) { log.trace("Deleting: '" + inKey + "'"); } client.delete(inKey); } /** * {@inheritDoc} */ public void deleteList(final String inKey) { this.delete(inKey); } /** * {@inheritDoc} */ public Map<String, Object> multiGet(final Collection<String> inKeys) { if (log.isTraceEnabled()) { log.trace("Getting bulk: " + inKeys.toString()); } return client.getBulk(inKeys); } /** * {@inheritDoc} */ public Map<String, ArrayList<Long>> multiGetList(final Collection<String> inKeys) { Map<String, Object> multiGetMap = this.multiGet(inKeys); Map<String, ArrayList<Long>> toReturn = new HashMap<String, ArrayList<Long>>(); for (String key : multiGetMap.keySet()) { // look up the value, convert it to an ArrayList and add it to the return try { ArrayList<Long> value = this.getListFromBytes(multiGetMap.get(key)); toReturn.put(key, value); } catch (IOException e) { log.error("Error getListFromBytes multiGetList memcached list with passed in value for key " + key + ". Exception : " + e.toString()); toReturn.put(key, null); // return null to force client to reload } } return toReturn; } /** * {@inheritDoc} */ public void addToTopOfList(final String inKey, final Long inValue) { ArrayList<Long> toAdd = new ArrayList<Long>(); toAdd.add(inValue); this.addToTopOfList(inKey, toAdd); } /** * {@inheritDoc} */ public void addToTopOfList(final String inKey, final List<Long> inValue) { if (inValue == null) { // cannot pass null to memcached log.warn("In addToTopOfList, attempting to pass in NULL to memcached for key " + inKey); return; } if (log.isTraceEnabled()) { StringBuilder allLongs = new StringBuilder(); for (Long lng : inValue) { allLongs.append(lng.toString()); allLongs.append(','); } // trim the last , just for clarity in the log if (allLongs.charAt(allLongs.length() - 1) == ',') { allLongs.deleteCharAt(allLongs.length() - 1); } log.trace("Prepending to list '" + inKey + "', values: " + allLongs.toString()); } // NOTE: memcached will NOT create a List if one does // not exist already, it will silently fail. // THIS IS WHAT WE WANT! // try to prepend data, NOTE the cas key is ignored // in ASCII protocol which we are using byte[] bytesToPrepend; try { bytesToPrepend = this.getBytesFromList(inValue); } catch (IOException e) { // problem getting the key .. set a null so the client goes // to the database and clear marker so next time we'll try again log.error("Unable to prepend LIST key " + inKey + " into memcached. Exception " + e.getMessage()); bytesToPrepend = null; } if (bytesToPrepend != null) { client.prepend(0, inKey, bytesToPrepend); } } /** * {@inheritDoc} */ public ArrayList<Long> getList(final String inKey) { return this.getList(inKey, this.getMaxListSize()); } /** * {@inheritDoc} */ public ArrayList<Long> getList(final String inKey, final int inMaximumEntries) { // This is how we modify a list when we find one in the cache. CASMutation<Object> mutation = new CASMutation<Object>() { // This is only invoked when a value actually exists. public Object getNewValue(final Object current) { // retrieve the list from the bytes stored in memcache ArrayList<Long> toReturn = new ArrayList<Long>(); try { toReturn = getListFromBytes(current); } catch (IOException e) { // problem getting the key .. return a null so the client goes // to the database log.error("Unable to retrieve LIST key " + inKey + " from memcached. Exception " + e.getMessage()); toReturn = null; } // check list size and trim if necessary if (toReturn != null && toReturn.size() > inMaximumEntries) { ArrayList<Long> trimmed = (ArrayList<Long>) toReturn.subList(0, inMaximumEntries - 1); // set the trimmed list back to memcached toReturn = trimmed; if (log.isInfoEnabled()) { log.info("Trimming list " + inKey + " to size of " + inMaximumEntries); } } try { return getBytesFromList(toReturn); } catch (IOException e) { // CAS timeout exceeded, return null because nothing was // stored in cache. log.error("Error in getList in getBytesFromList. Key " + inKey + " Exception " + e.toString()); return null; } } }; // The mutator who'll do all the low-level stuff. SerializingTranscoder transcoder = new SerializingTranscoder(); CASMutator<Object> mutator = new CASMutator<Object>(client, transcoder); // This returns whatever value was successfully stored within the // cache -- either the initial list as above, or a mutated existing // one try { // The initial value -- only used when there's no list stored under // the key. byte[] initialValue = null; return getListFromBytes(mutator.cas(inKey, initialValue, MAX_EXPIRATION_TIME, mutation)); } catch (Exception e) { // error, return null because nothing was stored in cache. log.error("Error in getList, general exception. Key " + inKey + " Exception " + e.toString()); return null; } } /** * Get the byte[] from a ArrayList<Long>. * * @param inListOfLongs the list of longs to convert * @return the byte[] representation of the ArrayList * @throws IOException thrown if any errors encountered */ protected byte[] getBytesFromList(final List<Long> inListOfLongs) throws IOException { if (inListOfLongs == null) { return null; } ByteArrayOutputStream bytes = new ByteArrayOutputStream(); DataOutputStream out = new DataOutputStream(bytes); byte[] toReturn = null; try { for (Long oneLong : inListOfLongs) { out.writeLong(oneLong); out.flush(); } toReturn = bytes.toByteArray(); } finally { out.close(); } return toReturn; } /** * Convert the memcached object into a List<Long>. * * @param inBytesOfLongs the byte[] to convert * @return the byte[] as List<Long>, null if not valid or empty bytes * @throws IOException thrown if any errors */ protected ArrayList<Long> getListFromBytes(final Object inBytesOfLongs) throws IOException { if (inBytesOfLongs == null) { return null; } ArrayList<Long> toReturn = new ArrayList<Long>(); ByteArrayInputStream bytes = new ByteArrayInputStream((byte[]) inBytesOfLongs); DataInputStream input = new DataInputStream(bytes); try { while (input.available() > 0) { toReturn.add(input.readLong()); } } finally { input.close(); } return toReturn; } /** * {@inheritDoc} */ public void removeFromList(final String inKey, final Long inValue) { if (log.isTraceEnabled()) { log.trace("Removing from list '" + inKey + "', value: " + inValue); } CASMutation<Object> mutation = new CASMutation<Object>() { // This is only invoked when a list already exists. public Object getNewValue(final Object current) { Object toReturn = current; List<Long> list; try { list = getListFromBytes(current); if (list != null) { list.remove(inValue); } // always convert back to byte[] to return to memcached toReturn = getBytesFromList(list); } catch (IOException e) { // we had a problem serializing from cache ... log an error // and continue assuming the app // will reload from database for the next reader log.error("Unable to retrieve key " + inKey + " from memcached. Not able to delete " + inValue + " from list."); } return toReturn; } }; SerializingTranscoder transcoder = new SerializingTranscoder(); CASMutator<Object> mutator = new CASMutator<Object>(client, transcoder); // This returns whatever value was successfully stored within the cache, // either the // initial list or a mutated existing one try { byte[] initialValue = null; mutator.cas(inKey, initialValue, MAX_EXPIRATION_TIME, mutation); } catch (Exception e) { log.error("Nothing to delete from memcached for key - " + inKey, e); } } /** * {@inheritDoc} */ public void removeFromLists(final List<String> inKeys, final List<Long> inValues) { Map<String, ArrayList<Long>> results = multiGetList(inKeys); for (final String key : results.keySet()) { if (log.isTraceEnabled()) { log.trace("Removing from list '" + key + "', values: " + inValues); } CASMutation<Object> mutation = new CASMutation<Object>() { // This is only invoked when a list already exists. public Object getNewValue(final Object current) { Object toReturn = current; List<Long> list; try { list = getListFromBytes(current); if (list != null) { list.removeAll(inValues); } // always convert back to byte[] to return to memcached toReturn = getBytesFromList(list); } catch (IOException e) { // we had a problem serializing from cache ... log an error // and continue assuming the app will reload from database for the next reader log.error("Unable to retrieve key " + key + " from memcached. Not able to delete " + inValues + " from list."); } return toReturn; } }; SerializingTranscoder transcoder = new SerializingTranscoder(); CASMutator<Object> mutator = new CASMutator<Object>(client, transcoder); // This returns whatever value was successfully stored within the cache, // either the initial list or a mutated existing one try { byte[] initialValue = null; mutator.cas(key, initialValue, MAX_EXPIRATION_TIME, mutation); } catch (Exception e) { log.error("Nothing to delete from memcached for key - " + key, e); } } } /** * {@inheritDoc} */ @SuppressWarnings("unchecked") public Set<Long> addToSet(final String inKey, final Long inValue) { if (log.isTraceEnabled()) { log.trace("Adding to set '" + inKey + "', value: " + inValue); } CASMutation<Object> mutation = new CASMutation<Object>() { // This is only invoked when a set already exists. public Object getNewValue(final Object current) { Set<Long> set = (Set<Long>) current; HashSet<Long> hashSet = new HashSet<Long>(set); hashSet.add(inValue); return hashSet; } }; SerializingTranscoder transcoder = new SerializingTranscoder(); CASMutator<Object> mutator = new CASMutator<Object>(client, transcoder); // This returns whatever value was successfully stored within the cache, // either the // initial set or a mutated existing one try { // Use null for initial value in this call. If set is not cached, leave it as null // so set is entirely initialized when requested. return (HashSet<Long>) mutator.cas(inKey, null, MAX_EXPIRATION_TIME, mutation); } catch (Exception e) { // CAS timeout exceeded, return empty set because nothing was // stored in cache. return new HashSet<Long>(); } } /** * {@inheritDoc} */ public void removeFromSet(final String inKey, final Long inValue) { if (log.isTraceEnabled()) { log.trace("Removing from set '" + inKey + "', value: " + inValue); } CASMutation<Object> mutation = new CASMutation<Object>() { // This is only invoked when a set already exists. @SuppressWarnings("unchecked") public Object getNewValue(final Object current) { Set<Long> set = (Set<Long>) current; HashSet<Long> hashSet = new HashSet<Long>(set); hashSet.remove(inValue); return hashSet; } }; SerializingTranscoder transcoder = new SerializingTranscoder(); CASMutator<Object> mutator = new CASMutator<Object>(client, transcoder); // This returns whatever value was successfully stored within the cache, // either the // initial set or a mutated existing one try { mutator.cas(inKey, new HashSet<Long>(), MAX_EXPIRATION_TIME, mutation); } catch (Exception e) { log.error("Nothing to delete from memcached for key - " + inKey, e); } } }