Java tutorial
/* * Copyright 2012-2014 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of * the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. */ package co.cask.tephra; import co.cask.tephra.inmemory.InMemoryTxSystemClient; import co.cask.tephra.metrics.TxMetricsCollector; import co.cask.tephra.persist.InMemoryTransactionStateStorage; import co.cask.tephra.persist.TransactionStateStorage; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import org.apache.hadoop.conf.Configuration; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import java.util.Collections; import java.util.concurrent.TimeUnit; /** * */ public class TransactionManagerTest extends TransactionSystemTest { static Configuration conf = new Configuration(); TransactionManager txManager = null; TransactionStateStorage txStateStorage = null; @Override protected TransactionSystemClient getClient() { return new InMemoryTxSystemClient(txManager); } @Override protected TransactionStateStorage getStateStorage() { return txStateStorage; } @Before public void before() { conf.setInt(TxConstants.Manager.CFG_TX_CLEANUP_INTERVAL, 0); // no cleanup thread // todo should create two sets of tests, one with LocalFileTxStateStorage and one with InMemoryTxStateStorage txStateStorage = new InMemoryTransactionStateStorage(); txManager = new TransactionManager(conf, txStateStorage, new TxMetricsCollector()); txManager.startAndWait(); } @After public void after() { txManager.stopAndWait(); } @Test public void testTransactionCleanup() throws Exception { conf.setInt(TxConstants.Manager.CFG_TX_CLEANUP_INTERVAL, 3); conf.setInt(TxConstants.Manager.CFG_TX_TIMEOUT, 2); // using a new tx manager that cleans up TransactionManager txm = new TransactionManager(conf, new InMemoryTransactionStateStorage(), new TxMetricsCollector()); txm.startAndWait(); try { Assert.assertEquals(0, txm.getInvalidSize()); Assert.assertEquals(0, txm.getCommittedSize()); // start a transaction and leave it open Transaction tx1 = txm.startShort(); // start a long running transaction and leave it open Transaction tx2 = txm.startLong(); Transaction tx3 = txm.startLong(); // start and commit a bunch of transactions for (int i = 0; i < 10; i++) { Transaction tx = txm.startShort(); Assert.assertTrue(txm.canCommit(tx, Collections.singleton(new byte[] { (byte) i }))); Assert.assertTrue(txm.commit(tx)); } // all of these should still be in the committed set Assert.assertEquals(0, txm.getInvalidSize()); Assert.assertEquals(10, txm.getCommittedSize()); // sleep longer than the cleanup interval TimeUnit.SECONDS.sleep(5); // transaction should now be invalid Assert.assertEquals(1, txm.getInvalidSize()); // run another transaction Transaction txx = txm.startShort(); // verify the exclude Assert.assertFalse(txx.isVisible(tx1.getTransactionId())); Assert.assertFalse(txx.isVisible(tx2.getTransactionId())); Assert.assertFalse(txx.isVisible(tx3.getTransactionId())); // try to commit the last transaction that was started Assert.assertTrue(txm.canCommit(txx, Collections.singleton(new byte[] { 0x0a }))); Assert.assertTrue(txm.commit(txx)); // now the committed change sets should be empty again Assert.assertEquals(0, txm.getCommittedSize()); // cannot commit transaction as it was timed out try { txm.canCommit(tx1, Collections.singleton(new byte[] { 0x11 })); Assert.fail(); } catch (TransactionNotInProgressException e) { // expected } txm.abort(tx1); // abort should have removed from invalid Assert.assertEquals(0, txm.getInvalidSize()); // run another bunch of transactions for (int i = 0; i < 10; i++) { Transaction tx = txm.startShort(); Assert.assertTrue(txm.canCommit(tx, Collections.singleton(new byte[] { (byte) i }))); Assert.assertTrue(txm.commit(tx)); } // none of these should still be in the committed set (tx2 is long-running). Assert.assertEquals(0, txm.getInvalidSize()); Assert.assertEquals(0, txm.getCommittedSize()); // commit tx2, abort tx3 Assert.assertTrue(txm.commit(tx2)); txm.abort(tx3); // none of these should still be in the committed set (tx2 is long-running). // Only tx3 is invalid list as it was aborted and is long-running. tx1 is short one and it rolled back its changes // so it should NOT be in invalid list Assert.assertEquals(1, txm.getInvalidSize()); Assert.assertEquals(tx3.getTransactionId(), (long) txm.getCurrentState().getInvalid().iterator().next()); Assert.assertEquals(1, txm.getExcludedListSize()); } finally { txm.stopAndWait(); } } @Test public void testLongTransactionCleanup() throws Exception { conf.setInt(TxConstants.Manager.CFG_TX_CLEANUP_INTERVAL, 3); conf.setInt(TxConstants.Manager.CFG_TX_LONG_TIMEOUT, 2); // using a new tx manager that cleans up TransactionManager txm = new TransactionManager(conf, new InMemoryTransactionStateStorage(), new TxMetricsCollector()); txm.startAndWait(); try { Assert.assertEquals(0, txm.getInvalidSize()); Assert.assertEquals(0, txm.getCommittedSize()); // start a long running transaction Transaction tx1 = txm.startLong(); Assert.assertEquals(0, txm.getInvalidSize()); Assert.assertEquals(0, txm.getCommittedSize()); // sleep longer than the cleanup interval TimeUnit.SECONDS.sleep(5); // transaction should now be invalid Assert.assertEquals(1, txm.getInvalidSize()); Assert.assertEquals(0, txm.getCommittedSize()); // cannot commit transaction as it was timed out try { txm.canCommit(tx1, Collections.singleton(new byte[] { 0x11 })); Assert.fail(); } catch (TransactionNotInProgressException e) { // expected } txm.abort(tx1); // abort should not remove long running transaction from invalid list Assert.assertEquals(1, txm.getInvalidSize()); } finally { txm.stopAndWait(); } } @Test public void testTruncateInvalid() throws Exception { InMemoryTransactionStateStorage storage = new InMemoryTransactionStateStorage(); Configuration testConf = new Configuration(conf); // No snapshots testConf.setLong(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL, -1); TransactionManager txm1 = new TransactionManager(testConf, storage, new TxMetricsCollector()); txm1.startAndWait(); TransactionManager txm2 = null; Transaction tx1; Transaction tx2; Transaction tx3; Transaction tx4; Transaction tx5; Transaction tx6; try { Assert.assertEquals(0, txm1.getInvalidSize()); // start a few transactions tx1 = txm1.startLong(); tx2 = txm1.startShort(); tx3 = txm1.startLong(); tx4 = txm1.startShort(); tx5 = txm1.startLong(); tx6 = txm1.startShort(); // invalidate tx1, tx2, tx5 and tx6 txm1.invalidate(tx1.getTransactionId()); txm1.invalidate(tx2.getTransactionId()); txm1.invalidate(tx5.getTransactionId()); txm1.invalidate(tx6.getTransactionId()); // tx1, tx2, tx5 and tx6 should be in invalid list Assert.assertEquals(ImmutableList.of(tx1.getTransactionId(), tx2.getTransactionId(), tx5.getTransactionId(), tx6.getTransactionId()), txm1.getCurrentState().getInvalid()); // remove tx1 and tx6 from invalid list Assert.assertTrue( txm1.truncateInvalidTx(ImmutableSet.of(tx1.getTransactionId(), tx6.getTransactionId()))); // only tx2 and tx5 should be in invalid list now Assert.assertEquals(ImmutableList.of(tx2.getTransactionId(), tx5.getTransactionId()), txm1.getCurrentState().getInvalid()); // removing in-progress transactions should not have any effect Assert.assertEquals(ImmutableSet.of(tx3.getTransactionId(), tx4.getTransactionId()), txm1.getCurrentState().getInProgress().keySet()); Assert.assertFalse( txm1.truncateInvalidTx(ImmutableSet.of(tx3.getTransactionId(), tx4.getTransactionId()))); // no change to in-progress Assert.assertEquals(ImmutableSet.of(tx3.getTransactionId(), tx4.getTransactionId()), txm1.getCurrentState().getInProgress().keySet()); // no change to invalid list Assert.assertEquals(ImmutableList.of(tx2.getTransactionId(), tx5.getTransactionId()), txm1.getCurrentState().getInvalid()); // Test transaction edit logs replay // Start another transaction manager without stopping txm1 so that snapshot does not get written, // and all logs can be replayed. txm2 = new TransactionManager(testConf, storage, new TxMetricsCollector()); txm2.startAndWait(); Assert.assertEquals(ImmutableList.of(tx2.getTransactionId(), tx5.getTransactionId()), txm2.getCurrentState().getInvalid()); Assert.assertEquals(ImmutableSet.of(tx3.getTransactionId(), tx4.getTransactionId()), txm2.getCurrentState().getInProgress().keySet()); } finally { txm1.stopAndWait(); if (txm2 != null) { txm2.stopAndWait(); } } } @Test public void testTruncateInvalidBeforeTime() throws Exception { InMemoryTransactionStateStorage storage = new InMemoryTransactionStateStorage(); Configuration testConf = new Configuration(conf); // No snapshots testConf.setLong(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL, -1); TransactionManager txm1 = new TransactionManager(testConf, storage, new TxMetricsCollector()); txm1.startAndWait(); TransactionManager txm2 = null; Transaction tx1; Transaction tx2; Transaction tx3; Transaction tx4; Transaction tx5; Transaction tx6; try { Assert.assertEquals(0, txm1.getInvalidSize()); // start a few transactions tx1 = txm1.startLong(); tx2 = txm1.startShort(); // Sleep so that transaction ids get generated a millisecond apart for assertion // TEPHRA-63 should eliminate the need to sleep TimeUnit.MILLISECONDS.sleep(1); long timeBeforeTx3 = System.currentTimeMillis(); tx3 = txm1.startLong(); tx4 = txm1.startShort(); TimeUnit.MILLISECONDS.sleep(1); long timeBeforeTx5 = System.currentTimeMillis(); tx5 = txm1.startLong(); tx6 = txm1.startShort(); // invalidate tx1, tx2, tx5 and tx6 txm1.invalidate(tx1.getTransactionId()); txm1.invalidate(tx2.getTransactionId()); txm1.invalidate(tx5.getTransactionId()); txm1.invalidate(tx6.getTransactionId()); // tx1, tx2, tx5 and tx6 should be in invalid list Assert.assertEquals(ImmutableList.of(tx1.getTransactionId(), tx2.getTransactionId(), tx5.getTransactionId(), tx6.getTransactionId()), txm1.getCurrentState().getInvalid()); // remove transactions before tx3 from invalid list Assert.assertTrue(txm1.truncateInvalidTxBefore(timeBeforeTx3)); // only tx5 and tx6 should be in invalid list now Assert.assertEquals(ImmutableList.of(tx5.getTransactionId(), tx6.getTransactionId()), txm1.getCurrentState().getInvalid()); // removing invalid transactions before tx5 should throw exception since tx3 and tx4 are in-progress Assert.assertEquals(ImmutableSet.of(tx3.getTransactionId(), tx4.getTransactionId()), txm1.getCurrentState().getInProgress().keySet()); try { txm1.truncateInvalidTxBefore(timeBeforeTx5); Assert.fail("Expected InvalidTruncateTimeException exception"); } catch (InvalidTruncateTimeException e) { // Expected exception } // no change to in-progress Assert.assertEquals(ImmutableSet.of(tx3.getTransactionId(), tx4.getTransactionId()), txm1.getCurrentState().getInProgress().keySet()); // no change to invalid list Assert.assertEquals(ImmutableList.of(tx5.getTransactionId(), tx6.getTransactionId()), txm1.getCurrentState().getInvalid()); // Test transaction edit logs replay // Start another transaction manager without stopping txm1 so that snapshot does not get written, // and all logs can be replayed. txm2 = new TransactionManager(testConf, storage, new TxMetricsCollector()); txm2.startAndWait(); Assert.assertEquals(ImmutableList.of(tx5.getTransactionId(), tx6.getTransactionId()), txm2.getCurrentState().getInvalid()); Assert.assertEquals(ImmutableSet.of(tx3.getTransactionId(), tx4.getTransactionId()), txm2.getCurrentState().getInProgress().keySet()); } finally { txm1.stopAndWait(); if (txm2 != null) { txm2.stopAndWait(); } } } }