Java tutorial
/* * Copyright 2008-2009 LinkedIn, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of * the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. */ package voldemort.scheduled; import static junit.framework.Assert.assertEquals; import static junit.framework.Assert.assertTrue; import static junit.framework.Assert.fail; import java.io.File; import java.io.StringReader; import java.util.Arrays; import java.util.Calendar; import java.util.Collection; import java.util.Date; import java.util.GregorianCalendar; import java.util.List; import java.util.Map; import java.util.Random; import org.apache.commons.io.FileDeleteStrategy; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; import voldemort.MockTime; import voldemort.TestUtils; import voldemort.VoldemortTestConstants; import voldemort.common.service.SchedulerService; import voldemort.server.VoldemortConfig; import voldemort.server.scheduler.DataCleanupJob; import voldemort.server.storage.ScanPermitWrapper; import voldemort.store.StorageEngine; import voldemort.store.StoreDefinition; import voldemort.store.bdb.BdbStorageConfiguration; import voldemort.store.retention.RetentionEnforcingStore; import voldemort.utils.ByteArray; import voldemort.utils.EventThrottler; import voldemort.utils.Props; import voldemort.utils.SystemTime; import voldemort.utils.Time; import voldemort.utils.Utils; import voldemort.versioning.VectorClock; import voldemort.versioning.Versioned; import voldemort.xml.StoreDefinitionsMapper; @RunWith(Parameterized.class) public class DataCleanupJobTest { private MockTime time; private StorageEngine<ByteArray, byte[], byte[]> engine; private File storeDir; private BdbStorageConfiguration bdbStorage; private boolean prefixPartitionId; public DataCleanupJobTest(boolean prefixPartitionId) { this.prefixPartitionId = prefixPartitionId; } @Parameters public static Collection<Object[]> modes() { Object[][] data = new Object[][] { { true }, { false } }; return Arrays.asList(data); } @Before public void setUp() throws Exception { time = new MockTime(); storeDir = TestUtils.createTempDir(); FileDeleteStrategy.FORCE.delete(storeDir); // lets use all the default values. Props props = new Props(); props.put("node.id", 1); props.put("voldemort.home", "test/common/voldemort/config"); VoldemortConfig voldemortConfig = new VoldemortConfig(props); voldemortConfig.setBdbCacheSize(1024 * 1024); voldemortConfig.setBdbOneEnvPerStore(true); voldemortConfig.setBdbDataDirectory(storeDir.toURI().getPath()); voldemortConfig.setBdbPrefixKeysWithPartitionId(prefixPartitionId); bdbStorage = new BdbStorageConfiguration(voldemortConfig); StoreDefinition defA = TestUtils.makeStoreDefinition("cleanupTestStore"); engine = bdbStorage.getStore(defA, TestUtils.makeSingleNodeRoutingStrategy()); } @After public void tearDown() throws Exception { try { if (engine != null) engine.close(); if (bdbStorage != null) bdbStorage.close(); } finally { FileDeleteStrategy.FORCE.delete(storeDir); } } @Test public void testCleanupFrequency() { SchedulerService scheduler = new SchedulerService(1, time); try { Date now = new Date(); // clean up will purge everything older than last 2 seconds Runnable cleanupJob = new DataCleanupJob<ByteArray, byte[], byte[]>(engine, new ScanPermitWrapper(1), 2 * Time.MS_PER_SECOND, SystemTime.INSTANCE, new EventThrottler(1), null); // and will run every 5 seconds starting now scheduler.schedule("cleanup-freq-test", cleanupJob, now, 5 * Time.MS_PER_SECOND); // load some data for (int i = 0; i < 10; i++) { ByteArray b = new ByteArray(Integer.toString(i).getBytes()); engine.put(b, new Versioned<byte[]>(b.get()), null); } // sleep for 2 seconds Thread.sleep(2 * Time.MS_PER_SECOND); // None of the keys should have been deleted, i.e data cleanup // should n't have run. for (int i = 0; i < 10; i++) { ByteArray b = new ByteArray(Integer.toString(i).getBytes()); List<Versioned<byte[]>> found = engine.get(b, null); assertTrue("Did not find key '" + i + "' in store!", found.size() > 0); } // wait till 4 seconds from start Thread.sleep(System.currentTimeMillis() - (now.getTime() + 4 * Time.MS_PER_SECOND)); // load some more data for (int i = 10; i < 20; i++) { ByteArray b = new ByteArray(Integer.toString(i).getBytes()); engine.put(b, new Versioned<byte[]>(b.get()), null); } // give time for data cleanup to finally run Thread.sleep(System.currentTimeMillis() - (now.getTime() + 6 * Time.MS_PER_SECOND)); // first batch of writes should have been deleted for (int i = 0; i < 10; i++) { ByteArray b = new ByteArray(Integer.toString(i).getBytes()); List<Versioned<byte[]>> found = engine.get(b, null); assertTrue("Expected key '" + i + "' to be deleted!", found.size() == 0); } // and later ones retained. for (int i = 10; i < 20; i++) { ByteArray b = new ByteArray(Integer.toString(i).getBytes()); List<Versioned<byte[]>> found = engine.get(b, null); assertTrue("Expected key '" + i + "' to be retained!", found.size() > 0); } } catch (Exception e) { } finally { scheduler.stop(); } } @Test public void testCleanupCleansUp() { time.setTime(123); put("a", "b", "c"); time.setTime(123 + Time.MS_PER_DAY + 1); put("d", "e", "f"); assertContains("a", "b", "c", "d", "e", "f"); // update a single item to bump its vector clock time put("a"); // now run cleanup new DataCleanupJob<ByteArray, byte[], byte[]>(engine, new ScanPermitWrapper(1), Time.MS_PER_DAY, time, new EventThrottler(1), null).run(); // Check that all the later keys are there AND the key updated later assertContains("a", "d", "e", "f"); } public void testCleanupStartTime() { // Make sure the default is always the next day. GregorianCalendar cal = new GregorianCalendar(); assertEquals("Default is not tomorrow", Utils.getDayOfTheWeekFromNow(1), (cal.get(Calendar.DAY_OF_WEEK) + 1) % 7); // When starting the server any day in the week from SUN to FRI and // targeting a saturday, should always start on the next saturday GregorianCalendar expectedStart = TestUtils.getCalendar(2012, Calendar.SEPTEMBER, 29, 0, 0, 0); Random rand = new Random(); for (int day = Calendar.SUNDAY; day <= Calendar.FRIDAY; day++) { GregorianCalendar serverStartTime = TestUtils.getCalendar(2012, Calendar.SEPTEMBER, 22 + day, rand.nextInt(24), rand.nextInt(60), rand.nextInt(60)); GregorianCalendar computedStart = Utils.getCalendarForNextRun(serverStartTime, Calendar.SATURDAY, 0); assertEquals( "Expected :" + expectedStart.getTimeInMillis() + " Computed: " + computedStart.getTimeInMillis(), expectedStart.getTimeInMillis(), computedStart.getTimeInMillis()); } // Targeting saturday, 00:00 and starting on a friday 23:59:59 should // start the next saturday GregorianCalendar serverStartTime = TestUtils.getCalendar(2012, Calendar.SEPTEMBER, 28, 23, 59, 59); GregorianCalendar computedStart = Utils.getCalendarForNextRun(serverStartTime, Calendar.SATURDAY, 0); assertEquals( "Expected :" + expectedStart.getTimeInMillis() + " Computed: " + computedStart.getTimeInMillis(), expectedStart.getTimeInMillis(), computedStart.getTimeInMillis()); // If we start past the start hour on the target day, it should start // the next week serverStartTime = TestUtils.getCalendar(2012, Calendar.SEPTEMBER, 29, 1, 0, 1); computedStart = Utils.getCalendarForNextRun(serverStartTime, Calendar.SATURDAY, 0); assertEquals(Calendar.SATURDAY, computedStart.get(Calendar.DAY_OF_WEEK)); assertEquals(serverStartTime.get(Calendar.DAY_OF_YEAR) + 7, computedStart.get(Calendar.DAY_OF_YEAR)); } private void runRetentionEnforcingStoreTest(boolean onlineDeletes) throws InterruptedException { time.setTime(System.currentTimeMillis()); StoreDefinition retentionStoreDef = new StoreDefinitionsMapper() .readStoreList(new StringReader(VoldemortTestConstants.getStoreDefinitionsWithRetentionXml())) .get(0); RetentionEnforcingStore store = new RetentionEnforcingStore(engine, retentionStoreDef, onlineDeletes, time); // do a bunch of puts store.put(new ByteArray("k1".getBytes()), new Versioned<byte[]>("v1".getBytes()), null); store.put(new ByteArray("k2".getBytes()), new Versioned<byte[]>("v2".getBytes()), null); long writeMs = System.currentTimeMillis(); // wait for a bit and then do more puts Thread.sleep(2000); store.put(new ByteArray("k3".getBytes()), new Versioned<byte[]>("v3".getBytes()), null); store.put(new ByteArray("k4".getBytes()), new Versioned<byte[]>("v4".getBytes()), null); // move time forward just enough such that some keys will have expired. time.setTime(writeMs + retentionStoreDef.getRetentionDays() * Time.MS_PER_DAY + 1); assertEquals("k1 should have expired", 0, store.get(new ByteArray("k1".getBytes()), null).size()); assertEquals("k2 should have expired", 0, store.get(new ByteArray("k2".getBytes()), null).size()); assertTrue("k3 should not have expired", store.get(new ByteArray("k3".getBytes()), null).size() > 0); assertTrue("k4 should not have expired", store.get(new ByteArray("k4".getBytes()), null).size() > 0); // get all with k1, k4 should return a map with k4 alone Map<ByteArray, List<Versioned<byte[]>>> getAllResult = store .getAll(Arrays.asList(new ByteArray("k1".getBytes()), new ByteArray("k4".getBytes())), null); assertEquals("map should contain one element only", 1, getAllResult.size()); assertEquals("k1 should not be present", false, getAllResult.containsKey(new ByteArray("k1".getBytes()))); assertEquals("k4 should be present", true, getAllResult.containsKey(new ByteArray("k4".getBytes()))); // if online deletes are not configured, we should see the deleted keys // in the base bdb store, so the datacleanup job can go and delete them assertEquals("k1 should be present", !onlineDeletes, engine.get(new ByteArray("k1".getBytes()), null).size() > 0); assertEquals("k2 should be present", !onlineDeletes, engine.get(new ByteArray("k2".getBytes()), null).size() > 0); // delete everything for next run engine.truncate(); } public void testRetentionEnforcingStore() throws InterruptedException { runRetentionEnforcingStoreTest(false); } public void testRetentionEnforcingStoreOnlineDeletes() throws InterruptedException { runRetentionEnforcingStoreTest(true); } private void put(String... items) { for (String item : items) { VectorClock clock = null; List<Versioned<byte[]>> found = engine.get(new ByteArray(item.getBytes()), null); if (found.size() == 0) { clock = new VectorClock(time.getMilliseconds()); } else if (found.size() == 1) { VectorClock oldClock = (VectorClock) found.get(0).getVersion(); clock = oldClock.incremented(0, time.getMilliseconds()); } else { fail("Found multiple versions."); } engine.put(new ByteArray(item.getBytes()), new Versioned<byte[]>(item.getBytes(), clock), null); } } private void assertContains(String... keys) { for (String key : keys) { List<Versioned<byte[]>> found = engine.get(new ByteArray(key.getBytes()), null); assertTrue("Did not find key '" + key + "' in store!", found.size() > 0); } } }