Java tutorial
// Copyright 2017 JanusGraph Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package org.janusgraph.diskstorage; import java.time.Duration; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicInteger; import org.janusgraph.diskstorage.keycolumnvalue.cache.KCVSCache; import org.janusgraph.diskstorage.log.kcvs.ExternalCachePersistor; import org.apache.commons.lang.StringUtils; import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalInterruptedException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Preconditions; import org.janusgraph.core.JanusGraphException; import org.janusgraph.diskstorage.indexing.IndexQuery; import org.janusgraph.diskstorage.indexing.IndexTransaction; import org.janusgraph.diskstorage.indexing.RawQuery; import org.janusgraph.diskstorage.keycolumnvalue.KeyIterator; import org.janusgraph.diskstorage.keycolumnvalue.KeyRangeQuery; import org.janusgraph.diskstorage.keycolumnvalue.KeySliceQuery; import org.janusgraph.diskstorage.keycolumnvalue.SliceQuery; import org.janusgraph.diskstorage.keycolumnvalue.StoreFeatures; import org.janusgraph.diskstorage.keycolumnvalue.StoreTransaction; import org.janusgraph.diskstorage.keycolumnvalue.cache.CacheTransaction; import org.janusgraph.diskstorage.util.BackendOperation; import org.janusgraph.diskstorage.util.BufferUtil; import org.janusgraph.graphdb.database.serialize.DataOutput; /** * Bundles all storage/index transactions and provides a proxy for some of their * methods for convenience. Also increases robustness of read call by attempting * read calls multiple times on failure. * * @author Matthias Broecheler (me@matthiasb.com) */ public class BackendTransaction implements LoggableTransaction { private static final Logger log = LoggerFactory.getLogger(BackendTransaction.class); public static final int MIN_TASKS_TO_PARALLELIZE = 2; //Assumes 64 bit key length as specified in IDManager public static final StaticBuffer EDGESTORE_MIN_KEY = BufferUtil.zeroBuffer(8); public static final StaticBuffer EDGESTORE_MAX_KEY = BufferUtil.oneBuffer(8); private final CacheTransaction storeTx; private final BaseTransactionConfig txConfig; private final StoreFeatures storeFeatures; private final KCVSCache edgeStore; private final KCVSCache indexStore; private final KCVSCache txLogStore; private final Duration maxReadTime; private final Executor threadPool; private final Map<String, IndexTransaction> indexTx; private boolean acquiredLock = false; private boolean cacheEnabled = true; public BackendTransaction(CacheTransaction storeTx, BaseTransactionConfig txConfig, StoreFeatures features, KCVSCache edgeStore, KCVSCache indexStore, KCVSCache txLogStore, Duration maxReadTime, Map<String, IndexTransaction> indexTx, Executor threadPool) { this.storeTx = storeTx; this.txConfig = txConfig; this.storeFeatures = features; this.edgeStore = edgeStore; this.indexStore = indexStore; this.txLogStore = txLogStore; this.maxReadTime = maxReadTime; this.indexTx = indexTx; this.threadPool = threadPool; } public boolean hasAcquiredLock() { return acquiredLock; } public StoreTransaction getStoreTransaction() { return storeTx; } public ExternalCachePersistor getTxLogPersistor() { return new ExternalCachePersistor(txLogStore, storeTx); } public BaseTransactionConfig getBaseTransactionConfig() { return txConfig; } public IndexTransaction getIndexTransaction(String index) { Preconditions.checkArgument(StringUtils.isNotBlank(index)); IndexTransaction itx = indexTx.get(index); Preconditions.checkNotNull(itx, "Unknown index: " + index); return itx; } public void disableCache() { this.cacheEnabled = false; } public void enableCache() { this.cacheEnabled = true; } public void commitStorage() throws BackendException { storeTx.commit(); } public Map<String, Throwable> commitIndexes() { Map<String, Throwable> exceptions = new HashMap<String, Throwable>(indexTx.size()); for (Map.Entry<String, IndexTransaction> txentry : indexTx.entrySet()) { try { txentry.getValue().commit(); } catch (Throwable e) { exceptions.put(txentry.getKey(), e); } } return exceptions; } @Override public void commit() throws BackendException { storeTx.commit(); for (IndexTransaction itx : indexTx.values()) itx.commit(); } /** * Rolls back all transactions and makes sure that this does not get cut short * by exceptions. If exceptions occur, the storage exception takes priority on re-throw. * @throws BackendException */ @Override public void rollback() throws BackendException { Throwable excep = null; for (IndexTransaction itx : indexTx.values()) { try { itx.rollback(); } catch (Throwable e) { excep = e; } } storeTx.rollback(); if (excep != null) { //throw any encountered index transaction rollback exceptions if (excep instanceof BackendException) throw (BackendException) excep; else throw new PermanentBackendException("Unexpected exception", excep); } } @Override public void logMutations(DataOutput out) { //Write storeTx.logMutations(out); for (Map.Entry<String, IndexTransaction> itx : indexTx.entrySet()) { out.writeObjectNotNull(itx.getKey()); itx.getValue().logMutations(out); } } /* ################################################### Convenience Write Methods */ /** * Applies the specified insertion and deletion mutations on the edge store to the provided key. * Both, the list of additions or deletions, may be empty or NULL if there is nothing to be added and/or deleted. * * @param key Key * @param additions List of entries (column + value) to be added * @param deletions List of columns to be removed */ public void mutateEdges(StaticBuffer key, List<Entry> additions, List<Entry> deletions) throws BackendException { edgeStore.mutateEntries(key, additions, deletions, storeTx); } /** * Applies the specified insertion and deletion mutations on the property index to the provided key. * Both, the list of additions or deletions, may be empty or NULL if there is nothing to be added and/or deleted. * * @param key Key * @param additions List of entries (column + value) to be added * @param deletions List of columns to be removed */ public void mutateIndex(StaticBuffer key, List<Entry> additions, List<Entry> deletions) throws BackendException { indexStore.mutateEntries(key, additions, deletions, storeTx); } /** * Acquires a lock for the key-column pair on the edge store which ensures that nobody else can take a lock on that * respective entry for the duration of this lock (but somebody could potentially still overwrite * the key-value entry without taking a lock). * The expectedValue defines the value expected to match the value at the time the lock is acquired (or null if it is expected * that the key-column pair does not exist). * <p/> * If this method is called multiple times with the same key-column pair in the same transaction, all but the first invocation are ignored. * <p/> * The lock has to be released when the transaction closes (commits or aborts). * * @param key Key on which to lock * @param column Column the column on which to lock */ public void acquireEdgeLock(StaticBuffer key, StaticBuffer column) throws BackendException { acquiredLock = true; edgeStore.acquireLock(key, column, null, storeTx); } public void acquireEdgeLock(StaticBuffer key, Entry entry) throws BackendException { acquiredLock = true; edgeStore.acquireLock(key, entry.getColumnAs(StaticBuffer.STATIC_FACTORY), entry.getValueAs(StaticBuffer.STATIC_FACTORY), storeTx); } /** * Acquires a lock for the key-column pair on the property index which ensures that nobody else can take a lock on that * respective entry for the duration of this lock (but somebody could potentially still overwrite * the key-value entry without taking a lock). * The expectedValue defines the value expected to match the value at the time the lock is acquired (or null if it is expected * that the key-column pair does not exist). * <p/> * If this method is called multiple times with the same key-column pair in the same transaction, all but the first invocation are ignored. * <p/> * The lock has to be released when the transaction closes (commits or aborts). * * @param key Key on which to lock * @param column Column the column on which to lock */ public void acquireIndexLock(StaticBuffer key, StaticBuffer column) throws BackendException { acquiredLock = true; indexStore.acquireLock(key, column, null, storeTx); } public void acquireIndexLock(StaticBuffer key, Entry entry) throws BackendException { acquiredLock = true; indexStore.acquireLock(key, entry.getColumnAs(StaticBuffer.STATIC_FACTORY), entry.getValueAs(StaticBuffer.STATIC_FACTORY), storeTx); } /* ################################################### Convenience Read Methods */ public EntryList edgeStoreQuery(final KeySliceQuery query) { return executeRead(new Callable<EntryList>() { @Override public EntryList call() throws Exception { return cacheEnabled ? edgeStore.getSlice(query, storeTx) : edgeStore.getSliceNoCache(query, storeTx); } @Override public String toString() { return "EdgeStoreQuery"; } }); } public Map<StaticBuffer, EntryList> edgeStoreMultiQuery(final List<StaticBuffer> keys, final SliceQuery query) { if (storeFeatures.hasMultiQuery()) { return executeRead(new Callable<Map<StaticBuffer, EntryList>>() { @Override public Map<StaticBuffer, EntryList> call() throws Exception { return cacheEnabled ? edgeStore.getSlice(keys, query, storeTx) : edgeStore.getSliceNoCache(keys, query, storeTx); } @Override public String toString() { return "MultiEdgeStoreQuery"; } }); } else { final Map<StaticBuffer, EntryList> results = new HashMap<StaticBuffer, EntryList>(keys.size()); if (threadPool == null || keys.size() < MIN_TASKS_TO_PARALLELIZE) { for (StaticBuffer key : keys) { results.put(key, edgeStoreQuery(new KeySliceQuery(key, query))); } } else { final CountDownLatch doneSignal = new CountDownLatch(keys.size()); final AtomicInteger failureCount = new AtomicInteger(0); EntryList[] resultArray = new EntryList[keys.size()]; for (int i = 0; i < keys.size(); i++) { threadPool.execute(new SliceQueryRunner(new KeySliceQuery(keys.get(i), query), doneSignal, failureCount, resultArray, i)); } try { doneSignal.await(); } catch (InterruptedException e) { throw new JanusGraphException("Interrupted while waiting for multi-query to complete", e); } if (failureCount.get() > 0) { throw new JanusGraphException("Could not successfully complete multi-query. " + failureCount.get() + " individual queries failed."); } for (int i = 0; i < keys.size(); i++) { assert resultArray[i] != null; results.put(keys.get(i), resultArray[i]); } } return results; } } private class SliceQueryRunner implements Runnable { final KeySliceQuery kq; final CountDownLatch doneSignal; final AtomicInteger failureCount; final Object[] resultArray; final int resultPosition; private SliceQueryRunner(KeySliceQuery kq, CountDownLatch doneSignal, AtomicInteger failureCount, Object[] resultArray, int resultPosition) { this.kq = kq; this.doneSignal = doneSignal; this.failureCount = failureCount; this.resultArray = resultArray; this.resultPosition = resultPosition; } @Override public void run() { try { List<Entry> result; result = edgeStoreQuery(kq); resultArray[resultPosition] = result; } catch (Exception e) { failureCount.incrementAndGet(); log.warn("Individual query in multi-transaction failed: ", e); } finally { doneSignal.countDown(); } } } public KeyIterator edgeStoreKeys(final SliceQuery sliceQuery) { if (!storeFeatures.hasScan()) throw new UnsupportedOperationException( "The configured storage backend does not support global graph operations - use Faunus instead"); return executeRead(new Callable<KeyIterator>() { @Override public KeyIterator call() throws Exception { return (storeFeatures.isKeyOrdered()) ? edgeStore.getKeys(new KeyRangeQuery(EDGESTORE_MIN_KEY, EDGESTORE_MAX_KEY, sliceQuery), storeTx) : edgeStore.getKeys(sliceQuery, storeTx); } @Override public String toString() { return "EdgeStoreKeys"; } }); } public KeyIterator edgeStoreKeys(final KeyRangeQuery range) { Preconditions.checkArgument(storeFeatures.hasOrderedScan(), "The configured storage backend does not support ordered scans"); return executeRead(new Callable<KeyIterator>() { @Override public KeyIterator call() throws Exception { return edgeStore.getKeys(range, storeTx); } @Override public String toString() { return "EdgeStoreKeys"; } }); } public EntryList indexQuery(final KeySliceQuery query) { return executeRead(new Callable<EntryList>() { @Override public EntryList call() throws Exception { return cacheEnabled ? indexStore.getSlice(query, storeTx) : indexStore.getSliceNoCache(query, storeTx); } @Override public String toString() { return "VertexIndexQuery"; } }); } public List<String> indexQuery(final String index, final IndexQuery query) { final IndexTransaction indexTx = getIndexTransaction(index); return executeRead(new Callable<List<String>>() { @Override public List<String> call() throws Exception { return indexTx.query(query); } @Override public String toString() { return "IndexQuery"; } }); } public Iterable<RawQuery.Result<String>> rawQuery(final String index, final RawQuery query) { final IndexTransaction indexTx = getIndexTransaction(index); return executeRead(new Callable<Iterable<RawQuery.Result<String>>>() { @Override public Iterable<RawQuery.Result<String>> call() throws Exception { return indexTx.query(query); } @Override public String toString() { return "RawQuery"; } }); } private final <V> V executeRead(Callable<V> exe) throws JanusGraphException { try { return BackendOperation.execute(exe, maxReadTime); } catch (JanusGraphException e) { // support traversal interruption // TODO: Refactor to allow direct propagation of underlying interrupt exception if (Thread.interrupted()) throw new TraversalInterruptedException(); throw e; } } }