Java tutorial
/* * Copyright 2012-2015 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of * the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. */ package co.cask.tephra; import co.cask.tephra.metrics.DefaultMetricsCollector; import co.cask.tephra.metrics.MetricsCollector; import co.cask.tephra.persist.NoOpTransactionStateStorage; import co.cask.tephra.persist.TransactionEdit; import co.cask.tephra.persist.TransactionLog; import co.cask.tephra.persist.TransactionLogReader; import co.cask.tephra.persist.TransactionSnapshot; import co.cask.tephra.persist.TransactionStateStorage; import co.cask.tephra.snapshot.SnapshotCodecProvider; import co.cask.tephra.util.TxUtils; import com.google.common.base.Objects; import com.google.common.base.Preconditions; import com.google.common.base.Stopwatch; import com.google.common.base.Throwables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.util.concurrent.AbstractService; import com.google.inject.Inject; import it.unimi.dsi.fastutil.longs.LongArrayList; import org.apache.hadoop.conf.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.OutputStream; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.NavigableMap; import java.util.Set; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; import javax.annotation.Nonnull; import javax.annotation.Nullable; /** * This is the central place to manage all active transactions in the system. * * A transaction consists of * <ul> * <li>A write pointer: This is the version used for all writes of that transaction.</li> * <li>A read pointer: All reads under the transaction use this as an upper bound for the version.</li> * <li>A set of excluded versions: These are the write versions of other transactions that must be excluded from * reads, because those transactions are still in progress, or they failed but couldn't be properly rolled back.</li> * </ul> * To use the transaction system, a client must follow this sequence of steps: * <ol> * <li>Request a new transaction.</li> * <li>Use the transaction to read and write datasets. Datasets are encouraged to cache the writes of the * transaction in memory, to reduce the cost of rollback in case the transaction fails. </li> * <li>Check whether the transaction has conflicts. For this, the set of change keys are submitted via canCommit(), * and the transaction manager verifies that none of these keys are in conflict with other transactions that * committed since the start of this transaction.</li> * <li>If the transaction has conflicts: * <ol> * <li>Roll back the changes in every dataset that was changed. This can happen in-memory if the * changes were cached.</li> * <li>Abort the transaction to remove it from the active transactions.</li> * </ol> * <li>If the transaction has no conflicts:</li> * <ol> * <li>Persist all datasets changes to storage.</li> * <li>Commit the transaction. This will repeat the conflict detection, because more overlapping transactions * may have committed since the first conflict check.</li> * <li>If the transaction has conflicts:</li> * <ol> * <li>Roll back the changes in every dataset that was changed. This is more expensive because * changes must be undone in persistent storage.</li> * <li>Abort the transaction to remove it from the active transactions.</li> * </ol> * </ol> * </ol> * Transactions may be short or long-running. A short transaction is started with a timeout, and if it is not * committed before that timeout, it is invalidated and excluded from future reads. A long-running transaction has * no timeout and will remain active until it is committed or aborted. Long transactions are typically used in * map/reduce jobs and can produce enormous amounts of changes. Therefore, long transactions do not participate in * conflict detection (they would almost always have conflicts). We also assume that the changes of long transactions * are not tracked, and therefore cannot be rolled back. Hence, when a long transaction is aborted, it remains in the * list of excluded transactions to make its writes invisible. */ public class TransactionManager extends AbstractService { // todo: optimize heavily private static final Logger LOG = LoggerFactory.getLogger(TransactionManager.class); // poll every 1 second to check whether a snapshot is needed private static final long SNAPSHOT_POLL_INTERVAL = 1000L; //poll every 10 second to emit metrics private static final long METRICS_POLL_INTERVAL = 10000L; private static final long[] NO_INVALID_TX = {}; // Transactions that are in progress, with their info. private final NavigableMap<Long, InProgressTx> inProgress = new ConcurrentSkipListMap<Long, InProgressTx>(); // the list of transactions that are invalid (not properly committed/aborted, or timed out) // TODO: explain usage of two arrays private final LongArrayList invalid = new LongArrayList(); private long[] invalidArray = NO_INVALID_TX; // todo: use moving array instead (use Long2ObjectMap<byte[]> in fastutil) // todo: should this be consolidated with inProgress? // commit time next writePointer -> changes made by this tx private final NavigableMap<Long, Set<ChangeId>> committedChangeSets = new ConcurrentSkipListMap<Long, Set<ChangeId>>(); // not committed yet private final Map<Long, Set<ChangeId>> committingChangeSets = Maps.newConcurrentMap(); private long readPointer; private long lastWritePointer; private MetricsCollector txMetricsCollector; private final TransactionStateStorage persistor; private final int cleanupInterval; private final int defaultTimeout; private final int defaultLongTimeout; private DaemonThreadExecutor cleanupThread = null; private volatile TransactionLog currentLog; // timestamp of the last completed snapshot private long lastSnapshotTime; // frequency in millis to perform snapshots private final long snapshotFrequencyInSeconds; // number of most recent snapshots to retain private final int snapshotRetainCount; private DaemonThreadExecutor snapshotThread; private DaemonThreadExecutor metricsThread; // lock guarding change of the current transaction log private final ReentrantReadWriteLock logLock = new ReentrantReadWriteLock(); private final Lock logReadLock = logLock.readLock(); private final Lock logWriteLock = logLock.writeLock(); // fudge factor (in milliseconds) used when interpreting transactions as LONG based on expiration // TODO: REMOVE WITH txnBackwardsCompatCheck() private final long longTimeoutTolerance; public TransactionManager(Configuration config) { this(config, new NoOpTransactionStateStorage(new SnapshotCodecProvider(config)), new DefaultMetricsCollector()); } @Inject public TransactionManager(Configuration conf, @Nonnull TransactionStateStorage persistor, MetricsCollector txMetricsCollector) { this.persistor = persistor; cleanupInterval = conf.getInt(TxConstants.Manager.CFG_TX_CLEANUP_INTERVAL, TxConstants.Manager.DEFAULT_TX_CLEANUP_INTERVAL); defaultTimeout = conf.getInt(TxConstants.Manager.CFG_TX_TIMEOUT, TxConstants.Manager.DEFAULT_TX_TIMEOUT); defaultLongTimeout = conf.getInt(TxConstants.Manager.CFG_TX_LONG_TIMEOUT, TxConstants.Manager.DEFAULT_TX_LONG_TIMEOUT); snapshotFrequencyInSeconds = conf.getLong(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL, TxConstants.Manager.DEFAULT_TX_SNAPSHOT_INTERVAL); // must always keep at least 1 snapshot snapshotRetainCount = Math.max(conf.getInt(TxConstants.Manager.CFG_TX_SNAPSHOT_RETAIN, TxConstants.Manager.DEFAULT_TX_SNAPSHOT_RETAIN), 1); // intentionally not using a constant, as this config should not be exposed // TODO: REMOVE WITH txnBackwardsCompatCheck() longTimeoutTolerance = conf.getLong("data.tx.long.timeout.tolerance", 10000); // this.txMetricsCollector = txMetricsCollector; this.txMetricsCollector.configure(conf); clear(); } private void clear() { invalid.clear(); invalidArray = NO_INVALID_TX; inProgress.clear(); committedChangeSets.clear(); committingChangeSets.clear(); lastWritePointer = 0; readPointer = 0; lastSnapshotTime = 0; } private boolean isStopping() { return State.STOPPING.equals(state()); } @Override public synchronized void doStart() { LOG.info("Starting transaction manager."); txMetricsCollector.start(); // start up the persistor persistor.startAndWait(); try { persistor.setupStorage(); } catch (IOException e) { Throwables.propagate(e); } // establish defaults in case there is no persistence clear(); // attempt to recover state from last run recoverState(); // start the periodic cleanup thread startCleanupThread(); startSnapshotThread(); startMetricsThread(); // initialize the WAL if we did not force a snapshot in recoverState() initLog(); // initialize next write pointer if needed if (lastWritePointer == 0) { lastWritePointer = getNextWritePointer(); readPointer = lastWritePointer; } notifyStarted(); } private void initLog() { if (currentLog == null) { try { currentLog = persistor.createLog(System.currentTimeMillis()); } catch (IOException ioe) { throw Throwables.propagate(ioe); } } } private void startCleanupThread() { if (cleanupInterval <= 0 || defaultTimeout <= 0) { return; } LOG.info("Starting periodic timed-out transaction cleanup every " + cleanupInterval + " seconds with default timeout of " + defaultTimeout + " seconds."); this.cleanupThread = new DaemonThreadExecutor("tx-clean-timeout") { @Override public void doRun() { cleanupTimedOutTransactions(); } @Override public long getSleepMillis() { return cleanupInterval * 1000; } }; cleanupThread.start(); } private void startSnapshotThread() { if (snapshotFrequencyInSeconds > 0) { LOG.info("Starting periodic snapshot thread, frequency = " + snapshotFrequencyInSeconds + " seconds, location = " + persistor.getLocation()); this.snapshotThread = new DaemonThreadExecutor("tx-snapshot") { @Override public void doRun() { long currentTime = System.currentTimeMillis(); if (lastSnapshotTime < (currentTime - snapshotFrequencyInSeconds * 1000)) { try { doSnapshot(false); } catch (IOException ioe) { LOG.error("Periodic snapshot failed!", ioe); } } } @Override protected void onShutdown() { // perform a final snapshot try { LOG.info("Writing final snapshot prior to shutdown"); doSnapshot(true); } catch (IOException ioe) { LOG.error("Failed performing final snapshot on shutdown", ioe); } } @Override public long getSleepMillis() { return SNAPSHOT_POLL_INTERVAL; } }; snapshotThread.start(); } } // Emits Transaction Data structures size as metrics private void startMetricsThread() { LOG.info("Starting periodic Metrics Emitter thread, frequency = " + METRICS_POLL_INTERVAL); this.metricsThread = new DaemonThreadExecutor("tx-metrics") { @Override public void doRun() { txMetricsCollector.gauge("committing.size", committingChangeSets.size()); txMetricsCollector.gauge("committed.size", committedChangeSets.size()); txMetricsCollector.gauge("inprogress.size", inProgress.size()); txMetricsCollector.gauge("invalid.size", invalidArray.length); } @Override protected void onShutdown() { // perform a final metrics emit txMetricsCollector.gauge("committing.size", committingChangeSets.size()); txMetricsCollector.gauge("committed.size", committedChangeSets.size()); txMetricsCollector.gauge("inprogress.size", inProgress.size()); txMetricsCollector.gauge("invalid.size", invalidArray.length); } @Override public long getSleepMillis() { return METRICS_POLL_INTERVAL; } }; metricsThread.start(); } private void cleanupTimedOutTransactions() { List<TransactionEdit> invalidEdits = null; this.logReadLock.lock(); try { synchronized (this) { if (!isRunning()) { return; } long currentTime = System.currentTimeMillis(); List<Long> timedOut = Lists.newArrayList(); for (Map.Entry<Long, InProgressTx> tx : inProgress.entrySet()) { long expiration = tx.getValue().getExpiration(); if (expiration >= 0L && currentTime > expiration) { // timed out, remember tx id (can't remove while iterating over entries) timedOut.add(tx.getKey()); LOG.info("Tx invalid list: added tx {} because of timeout", tx.getKey()); } else if (expiration < 0) { LOG.warn( "Transaction {} has negative expiration time {}. Likely cause is the transaction was not " + "migrated correctly, this transaction will be expired immediately", tx.getKey(), expiration); timedOut.add(tx.getKey()); } } if (!timedOut.isEmpty()) { invalidEdits = Lists.newArrayListWithCapacity(timedOut.size()); invalid.addAll(timedOut); for (long tx : timedOut) { committingChangeSets.remove(tx); inProgress.remove(tx); invalidEdits.add(TransactionEdit.createInvalid(tx)); } // todo: find a more efficient way to keep this sorted. Could it just be an array? Collections.sort(invalid); invalidArray = invalid.toLongArray(); LOG.info("Invalidated {} transactions due to timeout.", timedOut.size()); } } if (invalidEdits != null) { appendToLog(invalidEdits); } } finally { this.logReadLock.unlock(); } } public synchronized TransactionSnapshot getSnapshot() throws IOException { TransactionSnapshot snapshot = null; if (!isRunning() && !isStopping()) { return null; } long now = System.currentTimeMillis(); // avoid duplicate snapshots at same timestamp if (now == lastSnapshotTime || (currentLog != null && now == currentLog.getTimestamp())) { try { TimeUnit.MILLISECONDS.sleep(1); } catch (InterruptedException ie) { } } // copy in memory state snapshot = getCurrentState(); LOG.debug("Starting snapshot of transaction state with timestamp {}", snapshot.getTimestamp()); LOG.debug("Returning snapshot of state: " + snapshot); return snapshot; } /** * Take a snapshot of the transaction state and serialize it into the given output stream. * @return whether a snapshot was taken. */ public boolean takeSnapshot(OutputStream out) throws IOException { TransactionSnapshot snapshot = getSnapshot(); if (snapshot != null) { persistor.writeSnapshot(out, snapshot); return true; } else { return false; } } private void doSnapshot(boolean closing) throws IOException { long snapshotTime = 0L; TransactionSnapshot snapshot = null; TransactionLog oldLog = null; try { this.logWriteLock.lock(); try { synchronized (this) { snapshot = getSnapshot(); if (snapshot == null && !closing) { return; } if (snapshot != null) { snapshotTime = snapshot.getTimestamp(); } // roll WAL oldLog = currentLog; if (!closing) { currentLog = persistor.createLog(snapshot.getTimestamp()); } } // there may not be an existing log on startup if (oldLog != null) { oldLog.close(); } } finally { this.logWriteLock.unlock(); } // save snapshot if (snapshot != null) { persistor.writeSnapshot(snapshot); lastSnapshotTime = snapshotTime; // clean any obsoleted snapshots and WALs long oldestRetainedTimestamp = persistor.deleteOldSnapshots(snapshotRetainCount); persistor.deleteLogsOlderThan(oldestRetainedTimestamp); } } catch (IOException ioe) { abortService("Snapshot (timestamp " + snapshotTime + ") failed due to: " + ioe.getMessage(), ioe); } } public synchronized TransactionSnapshot getCurrentState() { return TransactionSnapshot.copyFrom(System.currentTimeMillis(), readPointer, lastWritePointer, invalid, inProgress, committingChangeSets, committedChangeSets); } public synchronized void recoverState() { try { TransactionSnapshot lastSnapshot = persistor.getLatestSnapshot(); // if we failed before a snapshot could complete, we might not have one to restore if (lastSnapshot != null) { restoreSnapshot(lastSnapshot); } // replay any WALs since the last snapshot Collection<TransactionLog> logs = persistor.getLogsSince(lastSnapshotTime); if (logs != null) { replayLogs(logs); } } catch (IOException e) { LOG.error("Unable to read back transaction state:", e); throw Throwables.propagate(e); } } /** * Restore the initial in-memory transaction state from a snapshot. */ private void restoreSnapshot(TransactionSnapshot snapshot) { LOG.info("Restoring transaction state from snapshot at " + snapshot.getTimestamp()); Preconditions.checkState(lastSnapshotTime == 0, "lastSnapshotTime has been set!"); Preconditions.checkState(readPointer == 0, "readPointer has been set!"); Preconditions.checkState(lastWritePointer == 0, "lastWritePointer has been set!"); Preconditions.checkState(invalid.isEmpty(), "invalid list should be empty!"); Preconditions.checkState(inProgress.isEmpty(), "inProgress map should be empty!"); Preconditions.checkState(committingChangeSets.isEmpty(), "committingChangeSets should be empty!"); Preconditions.checkState(committedChangeSets.isEmpty(), "committedChangeSets should be empty!"); LOG.info("Restoring snapshot of state: " + snapshot); lastSnapshotTime = snapshot.getTimestamp(); readPointer = snapshot.getReadPointer(); lastWritePointer = snapshot.getWritePointer(); invalid.addAll(snapshot.getInvalid()); inProgress.putAll( txnBackwardsCompatCheck(defaultLongTimeout, longTimeoutTolerance, snapshot.getInProgress())); committingChangeSets.putAll(snapshot.getCommittingChangeSets()); committedChangeSets.putAll(snapshot.getCommittedChangeSets()); } /** * Check if in-progress transactions need to be migrated to have expiration time and type, if so do the migration. * This is required for backwards compatibility, when long running transactions were represented * with expiration time -1. This can be removed when we stop supporting SnapshotCodec version 1. */ public static Map<Long, InProgressTx> txnBackwardsCompatCheck(int defaultLongTimeout, long longTimeoutTolerance, Map<Long, InProgressTx> inProgress) { for (Map.Entry<Long, InProgressTx> entry : inProgress.entrySet()) { long writePointer = entry.getKey(); long expiration = entry.getValue().getExpiration(); // LONG transactions will either have a negative expiration or expiration set to the long timeout // use a fudge factor on the expiration check, since expiraton is set based on system time, not the write pointer if (entry.getValue().getType() == null && (expiration < 0 || (getTxExpirationFromWritePointer(writePointer, defaultLongTimeout) - expiration < longTimeoutTolerance))) { // handle null expiration long newExpiration = getTxExpirationFromWritePointer(writePointer, defaultLongTimeout); InProgressTx compatTx = new InProgressTx(entry.getValue().getVisibilityUpperBound(), newExpiration, TransactionType.LONG, entry.getValue().getCheckpointWritePointers()); entry.setValue(compatTx); } else if (entry.getValue().getType() == null) { InProgressTx compatTx = new InProgressTx(entry.getValue().getVisibilityUpperBound(), entry.getValue().getExpiration(), TransactionType.SHORT, entry.getValue().getCheckpointWritePointers()); entry.setValue(compatTx); } } return inProgress; } /** * Resets the state of the transaction manager. */ public void resetState() { this.logWriteLock.lock(); try { // Take a snapshot before resetting the state, for debugging purposes doSnapshot(false); // Clear the state clear(); // Take another snapshot: if no snapshot is taken after clearing the state // and the manager is restarted, we will recover from the snapshot taken // before resetting the state, which would be really bad // This call will also init a new WAL doSnapshot(false); } catch (IOException e) { LOG.error("Snapshot failed when resetting state!", e); e.printStackTrace(); } finally { this.logWriteLock.unlock(); } } /** * Replay all logged edits from the given transaction logs. */ private void replayLogs(Collection<TransactionLog> logs) { for (TransactionLog log : logs) { LOG.info("Replaying edits from transaction log " + log.getName()); int editCnt = 0; try { TransactionLogReader reader = log.getReader(); // reader may be null in the case of an empty file if (reader == null) { continue; } TransactionEdit edit = null; while ((edit = reader.next()) != null) { editCnt++; switch (edit.getState()) { case INPROGRESS: long expiration = edit.getExpiration(); TransactionType type = edit.getType(); // Check if transaction needs to be migrated to have expiration and type. Previous version of // long running transactions were represented with expiration time as -1. // This can be removed when we stop supporting TransactionEditCodecV2. if (expiration < 0) { expiration = getTxExpirationFromWritePointer(edit.getWritePointer(), defaultLongTimeout); type = TransactionType.LONG; } else if (type == null) { type = TransactionType.SHORT; } addInProgressAndAdvance(edit.getWritePointer(), edit.getVisibilityUpperBound(), expiration, type); break; case COMMITTING: addCommittingChangeSet(edit.getWritePointer(), edit.getChanges()); break; case COMMITTED: // TODO: need to reconcile usage of transaction id v/s write pointer TEPHRA-140 long transactionId = edit.getWritePointer(); long[] checkpointPointers = edit.getCheckpointPointers(); long writePointer = checkpointPointers == null || checkpointPointers.length == 0 ? transactionId : checkpointPointers[checkpointPointers.length - 1]; doCommit(transactionId, writePointer, edit.getChanges(), edit.getCommitPointer(), edit.getCanCommit()); break; case INVALID: doInvalidate(edit.getWritePointer()); break; case ABORTED: type = edit.getType(); // Check if transaction edit needs to be migrated to have type. Previous versions of // ABORTED edits did not contain type. // This can be removed when we stop supporting TransactionEditCodecV2. if (type == null) { InProgressTx inProgressTx = inProgress.get(edit.getWritePointer()); if (inProgressTx != null) { type = inProgressTx.getType(); } else { // If transaction is not in-progress, then it has either been already aborted or invalidated. // We cannot determine the transaction's state based on current information, to be safe invalidate it. LOG.warn( "Invalidating transaction {} as it's type cannot be determined during replay", edit.getWritePointer()); doInvalidate(edit.getWritePointer()); break; } } doAbort(edit.getWritePointer(), edit.getCheckpointPointers(), type); break; case TRUNCATE_INVALID_TX: if (edit.getTruncateInvalidTxTime() != 0) { doTruncateInvalidTxBefore(edit.getTruncateInvalidTxTime()); } else { doTruncateInvalidTx(edit.getTruncateInvalidTx()); } break; case CHECKPOINT: doCheckpoint(edit.getWritePointer(), edit.getParentWritePointer()); break; default: // unknown type! throw new IllegalArgumentException("Invalid state for WAL entry: " + edit.getState()); } } } catch (IOException ioe) { throw Throwables.propagate(ioe); } catch (InvalidTruncateTimeException e) { throw Throwables.propagate(e); } LOG.info("Read " + editCnt + " edits from log " + log.getName()); } } @Override public void doStop() { Stopwatch timer = new Stopwatch().start(); LOG.info("Shutting down gracefully..."); // signal the cleanup thread to stop if (cleanupThread != null) { cleanupThread.shutdown(); try { cleanupThread.join(30000L); } catch (InterruptedException ie) { LOG.warn("Interrupted waiting for cleanup thread to stop"); Thread.currentThread().interrupt(); } } if (metricsThread != null) { metricsThread.shutdown(); try { metricsThread.join(30000L); } catch (InterruptedException ie) { LOG.warn("Interrupted waiting for cleanup thread to stop"); Thread.currentThread().interrupt(); } } if (snapshotThread != null) { // this will trigger a final snapshot on stop snapshotThread.shutdown(); try { snapshotThread.join(30000L); } catch (InterruptedException ie) { LOG.warn("Interrupted waiting for snapshot thread to stop"); Thread.currentThread().interrupt(); } } persistor.stopAndWait(); txMetricsCollector.stop(); timer.stop(); LOG.info("Took " + timer + " to stop"); notifyStopped(); } /** * Immediately shuts down the service, without going through the normal close process. * @param message A message describing the source of the failure. * @param error Any exception that caused the failure. */ private void abortService(String message, Throwable error) { if (isRunning()) { LOG.error("Aborting transaction manager due to: " + message, error); notifyFailed(error); } } private void ensureAvailable() { Preconditions.checkState(isRunning(), "Transaction Manager is not running."); } /** * Start a short transaction with the default timeout. */ public Transaction startShort() { return startShort(defaultTimeout); } /** * Start a short transaction with a given timeout. * @param timeoutInSeconds the time out period in seconds. */ public Transaction startShort(int timeoutInSeconds) { Preconditions.checkArgument(timeoutInSeconds > 0, "timeout must be positive but is %s", timeoutInSeconds); txMetricsCollector.rate("start.short"); Stopwatch timer = new Stopwatch().start(); long expiration = getTxExpiration(timeoutInSeconds); Transaction tx = startTx(expiration, TransactionType.SHORT); txMetricsCollector.histogram("start.short.latency", (int) timer.elapsedMillis()); return tx; } private static long getTxExpiration(long timeoutInSeconds) { long currentTime = System.currentTimeMillis(); return currentTime + TimeUnit.SECONDS.toMillis(timeoutInSeconds); } public static long getTxExpirationFromWritePointer(long writePointer, long timeoutInSeconds) { return writePointer / TxConstants.MAX_TX_PER_MS + TimeUnit.SECONDS.toMillis(timeoutInSeconds); } private long getNextWritePointer() { // We want to align tx ids with current time. We assume that tx ids are sequential, but not less than // System.currentTimeMillis() * MAX_TX_PER_MS. return Math.max(lastWritePointer + 1, System.currentTimeMillis() * TxConstants.MAX_TX_PER_MS); } /** * Start a long transaction. Long transactions and do not participate in conflict detection. Also, aborting a long * transaction moves it to the invalid list because we assume that its writes cannot be rolled back. */ public Transaction startLong() { txMetricsCollector.rate("start.long"); Stopwatch timer = new Stopwatch().start(); long expiration = getTxExpiration(defaultLongTimeout); Transaction tx = startTx(expiration, TransactionType.LONG); txMetricsCollector.histogram("start.long.latency", (int) timer.elapsedMillis()); return tx; } private Transaction startTx(long expiration, TransactionType type) { Transaction tx = null; long txid; // guard against changes to the transaction log while processing this.logReadLock.lock(); try { synchronized (this) { ensureAvailable(); txid = getNextWritePointer(); tx = createTransaction(txid, type); addInProgressAndAdvance(tx.getTransactionId(), tx.getVisibilityUpperBound(), expiration, type); } // appending to WAL out of global lock for concurrent performance // we should still be able to arrive at the same state even if log entries are out of order appendToLog(TransactionEdit.createStarted(tx.getTransactionId(), tx.getVisibilityUpperBound(), expiration, type)); } finally { this.logReadLock.unlock(); } return tx; } private void addInProgressAndAdvance(long writePointer, long visibilityUpperBound, long expiration, TransactionType type) { inProgress.put(writePointer, new InProgressTx(visibilityUpperBound, expiration, type)); advanceWritePointer(writePointer); } private void advanceWritePointer(long writePointer) { // don't move the write pointer back if we have out of order transaction log entries if (writePointer > lastWritePointer) { lastWritePointer = writePointer; } } public boolean canCommit(Transaction tx, Collection<byte[]> changeIds) throws TransactionNotInProgressException { txMetricsCollector.rate("canCommit"); Stopwatch timer = new Stopwatch().start(); if (inProgress.get(tx.getTransactionId()) == null) { // invalid transaction, either this has timed out and moved to invalid, or something else is wrong. if (invalid.contains(tx.getTransactionId())) { throw new TransactionNotInProgressException(String.format( "canCommit() is called for transaction %d that is not in progress (it is known to be invalid)", tx.getTransactionId())); } else { throw new TransactionNotInProgressException(String.format( "canCommit() is called for transaction %d that is not in progress", tx.getTransactionId())); } } Set<ChangeId> set = Sets.newHashSetWithExpectedSize(changeIds.size()); for (byte[] change : changeIds) { set.add(new ChangeId(change)); } if (hasConflicts(tx, set)) { return false; } // guard against changes to the transaction log while processing this.logReadLock.lock(); try { synchronized (this) { ensureAvailable(); addCommittingChangeSet(tx.getTransactionId(), set); } appendToLog(TransactionEdit.createCommitting(tx.getTransactionId(), set)); } finally { this.logReadLock.unlock(); } txMetricsCollector.histogram("canCommit.latency", (int) timer.elapsedMillis()); return true; } private void addCommittingChangeSet(long writePointer, Set<ChangeId> changes) { committingChangeSets.put(writePointer, changes); } public boolean commit(Transaction tx) throws TransactionNotInProgressException { txMetricsCollector.rate("commit"); Stopwatch timer = new Stopwatch().start(); Set<ChangeId> changeSet = null; boolean addToCommitted = true; long commitPointer; // guard against changes to the transaction log while processing this.logReadLock.lock(); try { synchronized (this) { ensureAvailable(); // we record commits at the first not-yet assigned transaction id to simplify clearing out change sets that // are no longer visible by any in-progress transactions commitPointer = lastWritePointer + 1; if (inProgress.get(tx.getTransactionId()) == null) { // invalid transaction, either this has timed out and moved to invalid, or something else is wrong. if (invalid.contains(tx.getTransactionId())) { throw new TransactionNotInProgressException( String.format("canCommit() is called for transaction %d that is not in progress " + "(it is known to be invalid)", tx.getTransactionId())); } else { throw new TransactionNotInProgressException( String.format("canCommit() is called for transaction %d that is not in progress", tx.getTransactionId())); } } // these should be atomic // NOTE: whether we succeed or not we don't need to keep changes in committing state: same tx cannot // be attempted to commit twice changeSet = committingChangeSets.remove(tx.getTransactionId()); if (changeSet != null) { // double-checking if there are conflicts: someone may have committed since canCommit check if (hasConflicts(tx, changeSet)) { return false; } } else { // no changes addToCommitted = false; } doCommit(tx.getTransactionId(), tx.getWritePointer(), changeSet, commitPointer, addToCommitted); } appendToLog(TransactionEdit.createCommitted(tx.getTransactionId(), changeSet, commitPointer, addToCommitted)); } finally { this.logReadLock.unlock(); } txMetricsCollector.histogram("commit.latency", (int) timer.elapsedMillis()); return true; } private void doCommit(long transactionId, long writePointer, Set<ChangeId> changes, long commitPointer, boolean addToCommitted) { // In case this method is called when loading a previous WAL, we need to remove the tx from these sets committingChangeSets.remove(transactionId); if (addToCommitted && !changes.isEmpty()) { // No need to add empty changes to the committed change sets, they will never trigger any conflict // Record the committed change set with the next writePointer as the commit time. // NOTE: we use current next writePointer as key for the map, hence we may have multiple txs changesets to be // stored under one key Set<ChangeId> changeIds = committedChangeSets.get(commitPointer); if (changeIds != null) { // NOTE: we modify the new set to prevent concurrent modification exception, as other threads (e.g. in // canCommit) use it unguarded changes.addAll(changeIds); } committedChangeSets.put(commitPointer, changes); } // remove from in-progress set, so that it does not get excluded in the future InProgressTx previous = inProgress.remove(transactionId); if (previous == null) { // tx was not in progress! perhaps it timed out and is invalid? try to remove it there. if (invalid.rem(transactionId)) { invalidArray = invalid.toLongArray(); LOG.info("Tx invalid list: removed committed tx {}", transactionId); } } // moving read pointer moveReadPointerIfNeeded(writePointer); // All committed change sets that are smaller than the earliest started transaction can be removed. // here we ignore transactions that have no timeout, they are long-running and don't participate in // conflict detection. // TODO: for efficiency, can we do this once per-log in replayLogs instead of once per edit? committedChangeSets.headMap(TxUtils.getFirstShortInProgress(inProgress)).clear(); } public void abort(Transaction tx) { // guard against changes to the transaction log while processing txMetricsCollector.rate("abort"); Stopwatch timer = new Stopwatch().start(); this.logReadLock.lock(); try { synchronized (this) { ensureAvailable(); doAbort(tx.getTransactionId(), tx.getCheckpointWritePointers(), tx.getType()); } appendToLog(TransactionEdit.createAborted(tx.getTransactionId(), tx.getType(), tx.getCheckpointWritePointers())); txMetricsCollector.histogram("abort.latency", (int) timer.elapsedMillis()); } finally { this.logReadLock.unlock(); } } private void doAbort(long writePointer, long[] checkpointWritePointers, TransactionType type) { committingChangeSets.remove(writePointer); if (type == TransactionType.LONG) { // Long running transactions cannot be aborted as their change sets are not saved, // and hence the changes cannot be rolled back. Invalidate the long running transaction instead. doInvalidate(writePointer); return; } // makes tx visible (assumes that all operations were rolled back) // remove from in-progress set, so that it does not get excluded in the future InProgressTx removed = inProgress.remove(writePointer); if (removed == null) { // tx was not in progress! perhaps it timed out and is invalid? try to remove it there. if (invalid.rem(writePointer)) { // remove any invalidated checkpoint pointers // this will only be present if the parent write pointer was also invalidated if (checkpointWritePointers != null) { for (int i = 0; i < checkpointWritePointers.length; i++) { invalid.rem(checkpointWritePointers[i]); } } invalidArray = invalid.toLongArray(); LOG.info("Tx invalid list: removed aborted tx {}", writePointer); // removed a tx from excludes: must move read pointer moveReadPointerIfNeeded(writePointer); } } else { // removed a tx from excludes: must move read pointer moveReadPointerIfNeeded(writePointer); } } public boolean invalidate(long tx) { // guard against changes to the transaction log while processing txMetricsCollector.rate("invalidate"); Stopwatch timer = new Stopwatch().start(); this.logReadLock.lock(); try { boolean success; synchronized (this) { ensureAvailable(); success = doInvalidate(tx); } appendToLog(TransactionEdit.createInvalid(tx)); txMetricsCollector.histogram("invalidate.latency", (int) timer.elapsedMillis()); return success; } finally { this.logReadLock.unlock(); } } private boolean doInvalidate(long writePointer) { Set<ChangeId> previousChangeSet = committingChangeSets.remove(writePointer); // remove from in-progress set, so that it does not get excluded in the future InProgressTx previous = inProgress.remove(writePointer); // This check is to prevent from invalidating committed transactions if (previous != null || previousChangeSet != null) { // add tx to invalids invalid.add(writePointer); if (previous == null) { LOG.debug("Invalidating tx {} in committing change sets but not in-progress", writePointer); } else { // invalidate any checkpoint write pointers LongArrayList childWritePointers = previous.getCheckpointWritePointers(); if (childWritePointers != null) { for (int i = 0; i < childWritePointers.size(); i++) { invalid.add(childWritePointers.get(i)); } } } LOG.info("Tx invalid list: added tx {} because of invalidate", writePointer); // todo: find a more efficient way to keep this sorted. Could it just be an array? Collections.sort(invalid); invalidArray = invalid.toLongArray(); if (previous != null && !previous.isLongRunning()) { // tx was short-running: must move read pointer moveReadPointerIfNeeded(writePointer); } return true; } return false; } /** * Removes the given transaction ids from the invalid list. * @param invalidTxIds transaction ids * @return true if invalid list got changed, false otherwise */ public boolean truncateInvalidTx(Set<Long> invalidTxIds) { // guard against changes to the transaction log while processing txMetricsCollector.rate("truncateInvalidTx"); Stopwatch timer = new Stopwatch().start(); this.logReadLock.lock(); try { boolean success; synchronized (this) { ensureAvailable(); success = doTruncateInvalidTx(invalidTxIds); } appendToLog(TransactionEdit.createTruncateInvalidTx(invalidTxIds)); txMetricsCollector.histogram("truncateInvalidTx.latency", (int) timer.elapsedMillis()); return success; } finally { this.logReadLock.unlock(); } } private boolean doTruncateInvalidTx(Set<Long> invalidTxIds) { LOG.info("Removing tx ids {} from invalid list", invalidTxIds); boolean success = invalid.removeAll(invalidTxIds); if (success) { invalidArray = invalid.toLongArray(); } return success; } /** * Removes all transaction ids started before the given time from invalid list. * @param time time in milliseconds * @return true if invalid list got changed, false otherwise * @throws InvalidTruncateTimeException if there are any in-progress transactions started before given time */ public boolean truncateInvalidTxBefore(long time) throws InvalidTruncateTimeException { // guard against changes to the transaction log while processing txMetricsCollector.rate("truncateInvalidTxBefore"); Stopwatch timer = new Stopwatch().start(); this.logReadLock.lock(); try { boolean success; synchronized (this) { ensureAvailable(); success = doTruncateInvalidTxBefore(time); } appendToLog(TransactionEdit.createTruncateInvalidTxBefore(time)); txMetricsCollector.histogram("truncateInvalidTxBefore.latency", (int) timer.elapsedMillis()); return success; } finally { this.logReadLock.unlock(); } } private boolean doTruncateInvalidTxBefore(long time) throws InvalidTruncateTimeException { LOG.info("Removing tx ids before {} from invalid list", time); long truncateWp = time * TxConstants.MAX_TX_PER_MS; // Check if there any in-progress transactions started earlier than truncate time if (inProgress.lowerKey(truncateWp) != null) { throw new InvalidTruncateTimeException( "Transactions started earlier than " + time + " are in-progress"); } // Find all invalid transactions earlier than truncateWp Set<Long> toTruncate = Sets.newHashSet(); for (long wp : invalid) { // invalid list is sorted, hence can stop as soon as we reach a wp >= truncateWp if (wp >= truncateWp) { break; } toTruncate.add(wp); } return doTruncateInvalidTx(toTruncate); } public Transaction checkpoint(Transaction originalTx) throws TransactionNotInProgressException { txMetricsCollector.rate("checkpoint"); Stopwatch timer = new Stopwatch().start(); Transaction checkpointedTx = null; long txId = originalTx.getTransactionId(); long newWritePointer = 0; // guard against changes to the transaction log while processing this.logReadLock.lock(); try { synchronized (this) { ensureAvailable(); // check that the parent tx is in progress InProgressTx parentTx = inProgress.get(txId); if (parentTx == null) { if (invalid.contains(txId)) { throw new TransactionNotInProgressException(String .format("Transaction %d is not in progress because it was invalidated", txId)); } else { throw new TransactionNotInProgressException( String.format("Transaction %d is not in progress", txId)); } } newWritePointer = getNextWritePointer(); doCheckpoint(newWritePointer, txId); // create a new transaction with the same read snapshot, plus the additional checkpoint write pointer // the same read snapshot is maintained to checkpointedTx = new Transaction(originalTx, newWritePointer, parentTx.getCheckpointWritePointers().toLongArray()); } // appending to WAL out of global lock for concurrent performance // we should still be able to arrive at the same state even if log entries are out of order appendToLog(TransactionEdit.createCheckpoint(newWritePointer, txId)); } finally { this.logReadLock.unlock(); } txMetricsCollector.histogram("checkpoint.latency", (int) timer.elapsedMillis()); return checkpointedTx; } private void doCheckpoint(long newWritePointer, long parentWritePointer) { InProgressTx existingTx = inProgress.get(parentWritePointer); existingTx.addCheckpointWritePointer(newWritePointer); advanceWritePointer(newWritePointer); } // hack for exposing important metric public int getExcludedListSize() { return invalid.size() + inProgress.size(); } /** * @return the size of invalid list */ public int getInvalidSize() { return this.invalid.size(); } int getCommittedSize() { return this.committedChangeSets.size(); } private boolean hasConflicts(Transaction tx, Set<ChangeId> changeIds) { if (changeIds.isEmpty()) { return false; } for (Map.Entry<Long, Set<ChangeId>> changeSet : committedChangeSets.entrySet()) { // If commit time is greater than tx read-pointer, // basically not visible but committed means "tx committed after given tx was started" if (changeSet.getKey() > tx.getTransactionId()) { if (overlap(changeSet.getValue(), changeIds)) { return true; } } } return false; } private boolean overlap(Set<ChangeId> a, Set<ChangeId> b) { // iterate over the smaller set, and check for every element in the other set if (a.size() > b.size()) { for (ChangeId change : b) { if (a.contains(change)) { return true; } } } else { for (ChangeId change : a) { if (b.contains(change)) { return true; } } } return false; } private void moveReadPointerIfNeeded(long committedWritePointer) { if (committedWritePointer > readPointer) { readPointer = committedWritePointer; } } /** * Creates a new Transaction. This method only get called from start transaction, which is already * synchronized. */ private Transaction createTransaction(long writePointer, TransactionType type) { // For holding the first in progress short transaction Id (with timeout >= 0). long firstShortTx = Transaction.NO_TX_IN_PROGRESS; LongArrayList inProgressIds = new LongArrayList(inProgress.size()); for (Map.Entry<Long, InProgressTx> entry : inProgress.entrySet()) { long txId = entry.getKey(); inProgressIds.add(txId); // add any checkpointed write pointers to the in-progress list LongArrayList childIds = entry.getValue().getCheckpointWritePointers(); if (childIds != null) { for (int i = 0; i < childIds.size(); i++) { inProgressIds.add(childIds.get(i)); } } if (firstShortTx == Transaction.NO_TX_IN_PROGRESS && !entry.getValue().isLongRunning()) { firstShortTx = txId; } } return new Transaction(readPointer, writePointer, invalidArray, inProgressIds.toLongArray(), firstShortTx, type); } private void appendToLog(TransactionEdit edit) { try { Stopwatch timer = new Stopwatch().start(); currentLog.append(edit); txMetricsCollector.rate("wal.append.count"); txMetricsCollector.histogram("wal.append.latency", (int) timer.elapsedMillis()); } catch (IOException ioe) { abortService("Error appending to transaction log", ioe); } } private void appendToLog(List<TransactionEdit> edits) { try { Stopwatch timer = new Stopwatch().start(); currentLog.append(edits); txMetricsCollector.rate("wal.append.count", edits.size()); txMetricsCollector.histogram("wal.append.latency", (int) timer.elapsedMillis()); } catch (IOException ioe) { abortService("Error appending to transaction log", ioe); } } /** * Called from the tx service every 10 seconds. * This hack is needed because current metrics system is not flexible when it comes to adding new metrics. */ public void logStatistics() { LOG.info("Transaction Statistics: write pointer = " + lastWritePointer + ", invalid = " + invalid.size() + ", in progress = " + inProgress.size() + ", committing = " + committingChangeSets.size() + ", committed = " + committedChangeSets.size()); } private abstract static class DaemonThreadExecutor extends Thread { private AtomicBoolean stopped = new AtomicBoolean(false); public DaemonThreadExecutor(String name) { super(name); setDaemon(true); } public void run() { try { while (!isInterrupted() && !stopped.get()) { doRun(); synchronized (stopped) { stopped.wait(getSleepMillis()); } } } catch (InterruptedException ie) { LOG.info("Interrupted thread " + getName()); } // perform any final cleanup onShutdown(); LOG.info("Exiting thread " + getName()); } public abstract void doRun(); protected abstract long getSleepMillis(); protected void onShutdown() { } public void shutdown() { if (stopped.compareAndSet(false, true)) { synchronized (stopped) { stopped.notifyAll(); } } } } /** * Represents some of the info on in-progress tx */ public static final class InProgressTx { /** the oldest in progress tx at the time of this tx start */ private final long visibilityUpperBound; private final long expiration; private final TransactionType type; private LongArrayList checkpointWritePointers = new LongArrayList(); public InProgressTx(long visibilityUpperBound, long expiration, TransactionType type) { this(visibilityUpperBound, expiration, type, new LongArrayList()); } public InProgressTx(long visibilityUpperBound, long expiration, TransactionType type, LongArrayList checkpointWritePointers) { this.visibilityUpperBound = visibilityUpperBound; this.expiration = expiration; this.type = type; this.checkpointWritePointers = checkpointWritePointers; } // For backwards compatibility when long running txns were represented with -1 expiration @Deprecated public InProgressTx(long visibilityUpperBound, long expiration) { this(visibilityUpperBound, expiration, null); } public long getVisibilityUpperBound() { return visibilityUpperBound; } public long getExpiration() { return expiration; } @Nullable public TransactionType getType() { return type; } public boolean isLongRunning() { if (type == null) { // for backwards compatibility when long running txns were represented with -1 expiration return expiration == -1; } return type == TransactionType.LONG; } public void addCheckpointWritePointer(long checkpointWritePointer) { checkpointWritePointers.add(checkpointWritePointer); } public LongArrayList getCheckpointWritePointers() { return checkpointWritePointers; } @Override public boolean equals(Object o) { if (o == null || !(o instanceof InProgressTx)) { return false; } if (this == o) { return true; } InProgressTx other = (InProgressTx) o; return Objects.equal(visibilityUpperBound, other.getVisibilityUpperBound()) && Objects.equal(expiration, other.getExpiration()) && Objects.equal(type, other.type) && Objects.equal(checkpointWritePointers, other.checkpointWritePointers); } @Override public int hashCode() { return Objects.hashCode(visibilityUpperBound, expiration, type, checkpointWritePointers); } @Override public String toString() { return Objects.toStringHelper(this).add("visibilityUpperBound", visibilityUpperBound) .add("expiration", expiration).add("type", type) .add("checkpointWritePointers", checkpointWritePointers).toString(); } } }