Java tutorial
/* * Copyright 2009 Martin Grotzke * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * */ package de.javakaffee.web.msm.integration; import static de.javakaffee.web.msm.integration.TestUtils.*; import static org.testng.Assert.*; import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import net.spy.memcached.MemcachedClient; import org.apache.catalina.Session; import org.apache.catalina.session.ManagerBase; import org.apache.catalina.startup.Embedded; import org.apache.http.HttpException; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.juli.logging.Log; import org.apache.juli.logging.LogFactory; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import com.thimbleware.jmemcached.CacheElement; import com.thimbleware.jmemcached.MemCacheDaemon; import de.javakaffee.web.msm.MemcachedSessionService; import de.javakaffee.web.msm.integration.TestUtils.Response; import de.javakaffee.web.msm.integration.TestUtils.SessionAffinityMode; /** * Integration test testing memcached failover. * * @author <a href="mailto:martin.grotzke@javakaffee.de">Martin Grotzke</a> * @version $Id$ */ public abstract class MemcachedFailoverIntegrationTest { private static final Log LOG = LogFactory.getLog(MemcachedFailoverIntegrationTest.class); private MemCacheDaemon<? extends CacheElement> _daemon1; private MemCacheDaemon<? extends CacheElement> _daemon2; private MemCacheDaemon<? extends CacheElement> _daemon3; private Embedded _tomcat1; private int _portTomcat1; private DefaultHttpClient _httpClient; private String _nodeId1; private String _nodeId2; private String _nodeId3; private InetSocketAddress _address1; private InetSocketAddress _address2; private InetSocketAddress _address3; @BeforeMethod public void setUp() throws Throwable { _portTomcat1 = 18888; _address1 = new InetSocketAddress("localhost", 21211); _daemon1 = createDaemon(_address1); _daemon1.start(); _address2 = new InetSocketAddress("localhost", 21212); _daemon2 = createDaemon(_address2); _daemon2.start(); _address3 = new InetSocketAddress("localhost", 21213); _daemon3 = createDaemon(_address3); _daemon3.start(); _nodeId1 = "n1"; _nodeId2 = "n2"; _nodeId3 = "n3"; try { final String memcachedNodes = toString(_nodeId1, _address1) + " " + toString(_nodeId2, _address2) + " " + toString(_nodeId3, _address3); _tomcat1 = getTestUtils().tomcatBuilder().port(_portTomcat1).sessionTimeout(10) .memcachedNodes(memcachedNodes).sticky(true).build(); _tomcat1.start(); } catch (final Throwable e) { LOG.error("could not start tomcat.", e); throw e; } _httpClient = new DefaultHttpClient(); } abstract TestUtils getTestUtils(); private String toString(final String nodeId, final InetSocketAddress address) { return nodeId + ":" + address.getHostName() + ":" + address.getPort(); } @AfterMethod public void tearDown() throws Exception { if (_daemon1.isRunning()) { _daemon1.stop(); } if (_daemon2.isRunning()) { _daemon2.stop(); } if (_daemon3.isRunning()) { _daemon3.stop(); } _tomcat1.stop(); _httpClient.getConnectionManager().shutdown(); } /** * Tests, that on a memcached failover sessions are relocated to another node and that * the session id reflects this. The session must no longer be available under the old * session id. */ @SuppressWarnings("unchecked") @Test(enabled = true, dataProviderClass = TestUtils.class, dataProvider = STICKYNESS_PROVIDER) public void testRelocateSession(final SessionAffinityMode sessionAffinity) throws Throwable { getManager(_tomcat1).setSticky(sessionAffinity.isSticky()); // we had a situation where no session was created, so let's take some break so that everything's up again Thread.sleep(200); final String sid1 = makeRequest(_httpClient, _portTomcat1, null); assertNotNull(sid1, "No session created."); final String firstNode = extractNodeId(sid1); assertNotNull(firstNode, "No node id encoded in session id."); final FailoverInfo info = getFailoverInfo(firstNode); info.activeNode.stop(); Thread.sleep(50); final String sid2 = makeRequest(_httpClient, _portTomcat1, sid1); final String secondNode = extractNodeId(sid2); assertNotSame(secondNode, firstNode, "First node again selected"); assertEquals(sid2, sid1.substring(0, sid1.indexOf("-") + 1) + secondNode, "Unexpected sessionId, sid1: " + sid1 + ", sid2: " + sid2); // we must get the same session back assertEquals(makeRequest(_httpClient, _portTomcat1, sid2), sid2, "We should keep the sessionId."); assertNotNull(getFailoverInfo(secondNode).activeNode.getCache().get(key(sid2))[0], "The session should exist in memcached."); // some more checks in sticky mode if (sessionAffinity.isSticky()) { final Session session = getManager(_tomcat1).findSession(sid2); assertNotNull(session, "Session not found by new id " + sid2); assertFalse(session.getNoteNames().hasNext(), "Some notes are set: " + toArray(session.getNoteNames())); } } /** * Tests that multiple memcached nodes can fail and backup/relocation handles this. */ @SuppressWarnings("unchecked") @Test(enabled = true, dataProviderClass = TestUtils.class, dataProvider = STICKYNESS_PROVIDER) public void testMultipleMemcachedNodesFailure(final SessionAffinityMode sessionAffinity) throws Throwable { getManager(_tomcat1).setSticky(sessionAffinity.isSticky()); // we had a situation where no session was created, so let's take some break so that everything's up again Thread.sleep(200); final String paramKey = "foo"; final String paramValue = "bar"; final String sid1 = post(_httpClient, _portTomcat1, null, paramKey, paramValue).getResponseSessionId(); assertNotNull(sid1, "No session created."); final String firstNode = extractNodeId(sid1); assertNotNull(firstNode, "No node id encoded in session id."); /* shutdown active and another memcached node */ final FailoverInfo info = getFailoverInfo(firstNode); info.activeNode.stop(); final Map.Entry<String, MemCacheDaemon<?>> otherNodeWithId = info.previousNode(); otherNodeWithId.getValue().stop(); Thread.sleep(100); final String sid2 = get(_httpClient, _portTomcat1, sid1).getResponseSessionId(); final String secondNode = extractNodeId(sid2); LOG.debug("Have secondNode " + secondNode); final String expectedNode = info.otherNodeExcept(otherNodeWithId.getKey()).getKey(); assertEquals(secondNode, expectedNode, "Unexpected nodeId: " + secondNode + "."); assertEquals(sid2, sid1.substring(0, sid1.indexOf("-") + 1) + expectedNode, "Unexpected sessionId, sid1: " + sid1 + ", sid2: " + sid2); // we must get the same session back final Response response2 = get(_httpClient, _portTomcat1, sid2); assertEquals(response2.getSessionId(), sid2, "We should keep the sessionId."); final MemCacheDaemon<?> activeNode = getFailoverInfo(secondNode).activeNode; assertNotNull(activeNode.getCache().get(key(sid2))[0], "The session should exist in memcached."); assertEquals(response2.get(paramKey), paramValue, "The session should still contain the previously stored value."); // some more checks in sticky mode if (sessionAffinity.isSticky()) { final Session session = getManager(_tomcat1).findSession(sid2); assertFalse(session.getNoteNames().hasNext(), "Some notes are set: " + toArray(session.getNoteNames())); } } /** * Tests that after a memcached failure (with only 1 memcached left) and reactivation the backup of the session is * stored again in the secondary memcached, so that the primary memcached can die and the session is still available. */ @Test(enabled = true) public void testSecondaryBackupForNonStickySessionAfterMemcachedFailover() throws Throwable { getManager(_tomcat1).setSticky(false); // we had a situation where no session was created, so let's take some break so that everything's up again Thread.sleep(200); final String paramKey = "foo"; final String paramValue = "bar"; final String sid1 = post(_httpClient, _portTomcat1, null, paramKey, paramValue).getResponseSessionId(); assertNotNull(sid1, "No session created."); final String firstNode = extractNodeId(sid1); assertNotNull(firstNode, "No node id encoded in session id."); /* shutdown other nodes */ LOG.info("-------------- stopping other nodes..."); final FailoverInfo info = getFailoverInfo(firstNode); for (final MemCacheDaemon<?> node : info.otherNodes.values()) { node.stop(); } Thread.sleep(100); /* make a request with only one memcached */ assertEquals(get(_httpClient, _portTomcat1, sid1).getSessionId(), sid1); Thread.sleep(300); // wait for the async processes to complete / be cancelleds /* now start the next node that shall get the backup again and make a request * that does not modify the session */ LOG.info("-------------- starting next node..."); info.nextNode().getValue().start(); waitForReconnect(getManager(_tomcat1).getMemcachedSessionService(), info.nextNode().getValue(), 5000); assertEquals(get(_httpClient, _portTomcat1, sid1).getSessionId(), sid1); Thread.sleep(300); // wait for the async processes to complete / be cancelleds /* now shutdown the active node so that the session is loaded from the secondary node */ LOG.info("-------------- stopping active node..."); info.activeNode.stop(); Thread.sleep(100); /* make the request and check that we still have all session data */ final String sid2 = get(_httpClient, _portTomcat1, sid1).getSessionId(); final String secondNode = extractNodeId(sid2); final String expectedNode = info.nextNode().getKey(); assertEquals(secondNode, expectedNode, "Unexpected nodeId: " + secondNode + "."); assertEquals(sid2, sid1.substring(0, sid1.indexOf("-") + 1) + expectedNode, "Unexpected sessionId, sid1: " + sid1 + ", sid2: " + sid2); // we must get the same session back final Response response2 = get(_httpClient, _portTomcat1, sid2); assertEquals(response2.getSessionId(), sid2, "We should keep the sessionId."); assertNotNull(getFailoverInfo(secondNode).activeNode.getCache().get(key(sid2))[0], "The session should exist in memcached."); assertEquals(response2.get(paramKey), paramValue, "The session should still contain the previously stored value."); } private void waitForReconnect(final MemcachedSessionService service, final MemCacheDaemon<?> value, final long timeToWait) throws InterruptedException { MemcachedClient client; InetSocketAddress serverAddress; try { final Method m = MemcachedSessionService.class.getDeclaredMethod("getMemcached"); m.setAccessible(true); client = (MemcachedClient) m.invoke(service); final Field field = MemCacheDaemon.class.getDeclaredField("addr"); field.setAccessible(true); serverAddress = (InetSocketAddress) field.get(value); } catch (final Exception e) { throw new RuntimeException(e); } waitForReconnect(client, serverAddress, timeToWait); } public void waitForReconnect(final MemcachedClient client, final InetSocketAddress serverAddressToCheck, final long timeToWait) throws InterruptedException, RuntimeException { final long start = System.currentTimeMillis(); while (System.currentTimeMillis() < start + timeToWait) { for (final SocketAddress address : client.getAvailableServers()) { if (address.equals(serverAddressToCheck)) { return; } } Thread.sleep(100); } throw new RuntimeException("MemcachedClient did not reconnect after " + timeToWait + " millis."); } private Set<String> toArray(final Iterator<String> noteNames) { final Set<String> result = new HashSet<String>(); while (noteNames.hasNext()) { result.add(noteNames.next()); } return result; } /** * Tests that the previous session id is kept when all memcached nodes fail. * * @throws Throwable */ @SuppressWarnings("unchecked") @Test(enabled = true) public void testAllMemcachedNodesFailure() throws Throwable { getManager(_tomcat1).setSticky(true); // we had a situation where no session was created, so let's take some break so that everything's up again Thread.sleep(200); final String sid1 = makeRequest(_httpClient, _portTomcat1, null); assertNotNull(sid1, "No session created."); /* shutdown all memcached nodes */ _daemon1.stop(); _daemon2.stop(); _daemon3.stop(); // wait a little bit Thread.sleep(200); final String sid2 = makeRequest(_httpClient, _portTomcat1, sid1); assertEquals(sid1, sid2, "SessionId changed."); assertNotNull(getSessions().get(sid1), "Session " + sid1 + " not existing."); final Session session = getManager(_tomcat1).findSession(sid2); assertFalse(session.getNoteNames().hasNext(), "Some notes are set: " + toArray(session.getNoteNames())); } @Test(enabled = true) public void testCookieNotSetWhenAllMemcachedsDownIssue40() throws IOException, HttpException, InterruptedException { getManager(_tomcat1).setSticky(true); // we had a situation where no session was created, so let's take some break so that everything's up again Thread.sleep(200); /* shutdown all memcached nodes */ _daemon1.stop(); _daemon2.stop(); _daemon3.stop(); final Response response1 = get(_httpClient, _portTomcat1, null); final String sessionId = response1.getSessionId(); assertNotNull(sessionId); assertNotNull(response1.getResponseSessionId()); final String nodeId = extractNodeId(response1.getResponseSessionId()); assertNull(nodeId, "NodeId should be null, but is " + nodeId + "."); final Response response2 = get(_httpClient, _portTomcat1, sessionId); assertEquals(response2.getSessionId(), sessionId, "SessionId changed"); assertNull(response2.getResponseSessionId()); } @Test(enabled = true, dataProviderClass = TestUtils.class, dataProvider = STICKYNESS_PROVIDER) public void testCookieNotSetWhenRegularMemcachedDownIssue40(final SessionAffinityMode sessionAffinity) throws Exception { /* reconfigure tomcat with failover node */ final String memcachedNodes = toString(_nodeId1, _address1) + " " + toString(_nodeId2, _address2); restartTomcat(memcachedNodes, _nodeId1); getManager(_tomcat1).setSticky(sessionAffinity.isSticky()); /* shutdown regular memcached node */ _daemon2.stop(); TestUtils.waitForReconnect(getService(_tomcat1).getMemcached(), 1, 1000l); final Response response1 = get(_httpClient, _portTomcat1, null); final String sessionId = response1.getSessionId(); assertNotNull(sessionId); assertNotNull(response1.getResponseSessionId()); final String nodeId = extractNodeId(response1.getResponseSessionId()); assertEquals(nodeId, _nodeId1); final Response response2 = get(_httpClient, _portTomcat1, sessionId); assertEquals(response2.getSessionId(), sessionId, "SessionId changed"); assertNull(response2.getResponseSessionId()); } @Test(enabled = true, dataProviderClass = TestUtils.class, dataProvider = STICKYNESS_PROVIDER) public void testReconfigureMemcachedNodesAtRuntimeFeature46(final SessionAffinityMode sessionAffinity) throws Exception { getManager(_tomcat1).setSticky(sessionAffinity.isSticky()); // we had a situation where no session was created, so let's take some break so that everything's up again Thread.sleep(200); /* reconfigure tomcat with only two memcached nodes */ final String memcachedNodes1 = toString(_nodeId1, _address1) + " " + toString(_nodeId2, _address2); restartTomcat(memcachedNodes1, _nodeId2); /* wait until everything's up and running... */ Thread.sleep(200); final Response response1 = get(_httpClient, _portTomcat1, null); final String sessionId1 = response1.getSessionId(); assertNotNull(sessionId1); assertEquals(extractNodeId(sessionId1), _nodeId1); /* reconfigure tomcat with only third memcached nodes and stop * the first one */ final String memcachedNodes2 = toString(_nodeId1, _address1) + " " + toString(_nodeId2, _address2) + " " + toString(_nodeId3, _address3); getManager(_tomcat1).setMemcachedNodes(memcachedNodes2); _daemon1.stop(); Thread.sleep(1000); /* Expect relocation to node3 */ final Response response2 = get(_httpClient, _portTomcat1, sessionId1); assertNotSame(response2.getSessionId(), sessionId1); final String sessionId2 = response2.getResponseSessionId(); assertNotNull(sessionId2); assertEquals(extractNodeId(sessionId2), _nodeId3); } @Test(enabled = true) public void testReconfigureFailoverNodesAtRuntimeFeature46() throws Exception { getManager(_tomcat1).setSticky(true); /* set failover nodes n2 and n3 */ getManager(_tomcat1).setFailoverNodes(_nodeId2 + " " + _nodeId3); /* wait for changes... */ Thread.sleep(200); final Response response1 = get(_httpClient, _portTomcat1, null); final String sessionId1 = response1.getSessionId(); assertNotNull(sessionId1); assertEquals(extractNodeId(sessionId1), _nodeId1); /* set failover nodes n1 and n2 */ getManager(_tomcat1).setFailoverNodes(_nodeId1 + " " + _nodeId2); /* wait for changes... */ Thread.sleep(200); // we need to use another http client, otherwise there's no response cookie. final Response response2 = get(new DefaultHttpClient(), _portTomcat1, null); final String sessionId2 = response2.getSessionId(); assertNotNull(sessionId2); assertEquals(extractNodeId(sessionId2), _nodeId3); } private void restartTomcat(final String memcachedNodes, final String failoverNodes) throws Exception { _tomcat1.stop(); Thread.sleep(500); _tomcat1 = getTestUtils().tomcatBuilder().port(_portTomcat1).sessionTimeout(10) .memcachedNodes(memcachedNodes).failoverNodes(failoverNodes).build(); _tomcat1.start(); } private Map<String, Session> getSessions() throws NoSuchFieldException, IllegalAccessException { final Field field = ManagerBase.class.getDeclaredField("sessions"); field.setAccessible(true); @SuppressWarnings("unchecked") final Map<String, Session> sessions = (Map<String, Session>) field.get(getManager(_tomcat1)); return sessions; } /* plain stupid */ private FailoverInfo getFailoverInfo(final String nodeId) { if (_nodeId1.equals(nodeId)) { return new FailoverInfo(_daemon1, asMap(_nodeId2, _daemon2, _nodeId3, _daemon3)); } else if (_nodeId2.equals(nodeId)) { return new FailoverInfo(_daemon2, asMap(_nodeId3, _daemon3, _nodeId1, _daemon1)); } else if (_nodeId3.equals(nodeId)) { return new FailoverInfo(_daemon3, asMap(_nodeId1, _daemon1, _nodeId2, _daemon2)); } throw new IllegalArgumentException("Node " + nodeId + " is not a valid node id."); } private Map<String, MemCacheDaemon<?>> asMap(final String nodeId1, final MemCacheDaemon<?> daemon1, final String nodeId2, final MemCacheDaemon<?> daemon2) { final Map<String, MemCacheDaemon<?>> result = new LinkedHashMap<String, MemCacheDaemon<?>>(2); result.put(nodeId1, daemon1); result.put(nodeId2, daemon2); return result; } static class FailoverInfo { MemCacheDaemon<?> activeNode; Map<String, MemCacheDaemon<?>> otherNodes; public FailoverInfo(final MemCacheDaemon<?> first, final Map<String, MemCacheDaemon<?>> otherNodes) { this.activeNode = first; this.otherNodes = otherNodes; } public Entry<String, MemCacheDaemon<?>> nextNode() { return otherNodes.entrySet().iterator().next(); } public Entry<String, MemCacheDaemon<?>> previousNode() { Entry<String, MemCacheDaemon<?>> last = null; for (final Entry<String, MemCacheDaemon<?>> entry : otherNodes.entrySet()) { last = entry; } return last; } public Entry<String, MemCacheDaemon<?>> otherNodeExcept(final String key) { for (final Map.Entry<String, MemCacheDaemon<?>> entry : otherNodes.entrySet()) { if (!entry.getKey().equals(key)) { return entry; } } throw new IllegalStateException(); } } }