Java tutorial
/* * Copyright 2012-2014 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of * the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. */ package co.cask.tephra.persist; import co.cask.tephra.ChangeId; import co.cask.tephra.TransactionManager; import co.cask.tephra.TransactionType; import co.cask.tephra.TxConstants; import co.cask.tephra.metrics.TxMetricsCollector; import co.cask.tephra.snapshot.DefaultSnapshotCodec; import co.cask.tephra.snapshot.SnapshotCodecProvider; import co.cask.tephra.snapshot.SnapshotCodecV4; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSortedSet; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.hadoop.conf.Configuration; import org.junit.Assert; import org.junit.ClassRule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import java.io.DataOutput; import java.io.File; import java.io.IOException; import java.util.Collection; import java.util.Map; import java.util.NavigableMap; import java.util.Set; import java.util.concurrent.TimeUnit; /** * Runs transaction persistence tests against the {@link LocalFileTransactionStateStorage} and * {@link LocalFileTransactionLog} implementations. */ public class LocalTransactionStateStorageTest extends AbstractTransactionStateStorageTest { @ClassRule public static TemporaryFolder tmpDir = new TemporaryFolder(); @Override protected Configuration getConfiguration(String testName) throws IOException { File testDir = tmpDir.newFolder(testName); Configuration conf = new Configuration(); conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_LOCAL_DIR, testDir.getAbsolutePath()); conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, SnapshotCodecV4.class.getName()); return conf; } @Override protected AbstractTransactionStateStorage getStorage(Configuration conf) { return new LocalFileTransactionStateStorage(conf, new SnapshotCodecProvider(conf), new TxMetricsCollector()); } // v2 TransactionEdit @SuppressWarnings("deprecation") private class TransactionEditV2 extends TransactionEdit { public TransactionEditV2(long writePointer, long visibilityUpperBound, State state, long expirationDate, Set<ChangeId> changes, long commitPointer, boolean canCommit, TransactionType type) { super(writePointer, visibilityUpperBound, state, expirationDate, changes, commitPointer, canCommit, type, null, 0L, 0L, null); } @Override public void write(DataOutput out) throws IOException { TransactionEditCodecs.encode(this, out, new TransactionEditCodecs.TransactionEditCodecV2()); } } // Note: this test cannot run in AbstractTransactionStateStorageTest, since SequenceFile throws exception saying // TransactionEditV2 is not TransactionEdit. Since the code path this test is verifying is the same path between // HDFS and Local Storage, having this only over here is fine. @SuppressWarnings("deprecation") @Test public void testLongTxnBackwardsCompatibility() throws Exception { Configuration conf = getConfiguration("testLongTxnBackwardsCompatibility"); // Use SnapshotCodec version 1 String latestSnapshotCodec = conf.get(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES); conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, DefaultSnapshotCodec.class.getName()); TransactionStateStorage storage = null; try { storage = getStorage(conf); storage.startAndWait(); // Create transaction snapshot and transaction edits with version when long running txns had -1 expiration. Collection<Long> invalid = Lists.newArrayList(); NavigableMap<Long, TransactionManager.InProgressTx> inProgress = Maps.newTreeMap(); long time1 = System.currentTimeMillis(); long wp1 = time1 * TxConstants.MAX_TX_PER_MS; inProgress.put(wp1, new TransactionManager.InProgressTx(wp1 - 5, -1L)); long time2 = time1 + 100; long wp2 = time2 * TxConstants.MAX_TX_PER_MS; inProgress.put(wp2, new TransactionManager.InProgressTx(wp2 - 50, time2 + 1000)); Map<Long, Set<ChangeId>> committing = Maps.newHashMap(); Map<Long, Set<ChangeId>> committed = Maps.newHashMap(); TransactionSnapshot snapshot = new TransactionSnapshot(time2, 0, wp2, invalid, inProgress, committing, committed); long time3 = time1 + 200; long wp3 = time3 * TxConstants.MAX_TX_PER_MS; TransactionEdit edit1 = new TransactionEditV2(wp3, wp3 - 10, TransactionEdit.State.INPROGRESS, -1L, null, 0L, false, null); long time4 = time1 + 300; long wp4 = time4 * TxConstants.MAX_TX_PER_MS; TransactionEdit edit2 = new TransactionEditV2(wp4, wp4 - 10, TransactionEdit.State.INPROGRESS, time4 + 1000, null, 0L, false, null); // write snapshot and transaction edit storage.writeSnapshot(snapshot); TransactionLog log = storage.createLog(time2); log.append(edit1); log.append(edit2); log.close(); // Start transaction manager conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, latestSnapshotCodec); long longTimeout = TimeUnit.SECONDS.toMillis(conf.getLong(TxConstants.Manager.CFG_TX_LONG_TIMEOUT, TxConstants.Manager.DEFAULT_TX_LONG_TIMEOUT)); TransactionManager txm = new TransactionManager(conf, storage, new TxMetricsCollector()); txm.startAndWait(); try { // Verify that the txns in old format were read correctly. // There should be four in-progress transactions, and no invalid transactions TransactionSnapshot snapshot1 = txm.getCurrentState(); Assert.assertEquals(ImmutableSortedSet.of(wp1, wp2, wp3, wp4), snapshot1.getInProgress().keySet()); verifyInProgress(snapshot1.getInProgress().get(wp1), TransactionType.LONG, time1 + longTimeout); verifyInProgress(snapshot1.getInProgress().get(wp2), TransactionType.SHORT, time2 + 1000); verifyInProgress(snapshot1.getInProgress().get(wp3), TransactionType.LONG, time3 + longTimeout); verifyInProgress(snapshot1.getInProgress().get(wp4), TransactionType.SHORT, time4 + 1000); Assert.assertEquals(0, snapshot1.getInvalid().size()); } finally { txm.stopAndWait(); } } finally { if (storage != null) { storage.stopAndWait(); } } } // Note: this test cannot run in AbstractTransactionStateStorageTest, since SequenceFile throws exception saying // TransactionEditV2 is not TransactionEdit. Since the code path this test is verifying is the same path between // HDFS and Local Storage, having this only over here is fine. @SuppressWarnings("deprecation") @Test public void testAbortEditBackwardsCompatibility() throws Exception { Configuration conf = getConfiguration("testAbortEditBackwardsCompatibility"); TransactionStateStorage storage = null; try { storage = getStorage(conf); storage.startAndWait(); // Create edits for transaction type addition to abort long time1 = System.currentTimeMillis(); long wp1 = time1 * TxConstants.MAX_TX_PER_MS; TransactionEdit edit1 = new TransactionEditV2(wp1, wp1 - 10, TransactionEdit.State.INPROGRESS, -1L, null, 0L, false, null); TransactionEdit edit2 = new TransactionEditV2(wp1, 0L, TransactionEdit.State.ABORTED, 0L, null, 0L, false, null); long time2 = time1 + 400; long wp2 = time2 * TxConstants.MAX_TX_PER_MS; TransactionEdit edit3 = new TransactionEditV2(wp2, wp2 - 10, TransactionEdit.State.INPROGRESS, time2 + 10000, null, 0L, false, null); TransactionEdit edit4 = new TransactionEditV2(wp2, 0L, TransactionEdit.State.INVALID, 0L, null, 0L, false, null); // Simulate case where we cannot determine txn state during abort TransactionEdit edit5 = new TransactionEditV2(wp2, 0L, TransactionEdit.State.ABORTED, 0L, null, 0L, false, null); // write snapshot and transaction edit TransactionLog log = storage.createLog(time1); log.append(edit1); log.append(edit2); log.append(edit3); log.append(edit4); log.append(edit5); log.close(); // Start transaction manager TransactionManager txm = new TransactionManager(conf, storage, new TxMetricsCollector()); txm.startAndWait(); try { // Verify that the txns in old format were read correctly. // Both transactions should be in invalid state TransactionSnapshot snapshot1 = txm.getCurrentState(); Assert.assertEquals(ImmutableList.of(wp1, wp2), snapshot1.getInvalid()); Assert.assertEquals(0, snapshot1.getInProgress().size()); Assert.assertEquals(0, snapshot1.getCommittedChangeSets().size()); Assert.assertEquals(0, snapshot1.getCommittingChangeSets().size()); } finally { txm.stopAndWait(); } } finally { if (storage != null) { storage.stopAndWait(); } } } private void verifyInProgress(TransactionManager.InProgressTx inProgressTx, TransactionType type, long expiration) throws Exception { Assert.assertEquals(type, inProgressTx.getType()); Assert.assertTrue(inProgressTx.getExpiration() == expiration); } }