Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.bigdata.dastor.service; import java.io.ByteArrayInputStream; import java.io.DataInputStream; import java.io.IOException; import java.lang.management.ManagementFactory; import java.net.InetAddress; import java.util.*; import java.util.concurrent.Callable; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import javax.management.MBeanServer; import javax.management.ObjectName; import org.apache.log4j.Logger; import org.apache.commons.lang.ArrayUtils; import com.bigdata.dastor.cfc.Collector; import com.bigdata.dastor.concurrent.StageManager; import com.bigdata.dastor.config.DatabaseDescriptor; import com.bigdata.dastor.db.*; import com.bigdata.dastor.dht.*; import com.bigdata.dastor.locator.AbstractReplicationStrategy; import com.bigdata.dastor.locator.TokenMetadata; import com.bigdata.dastor.net.IAsyncResult; import com.bigdata.dastor.net.Message; import com.bigdata.dastor.net.MessagingService; import com.bigdata.dastor.thrift.ConsistencyLevel; import com.bigdata.dastor.thrift.InvalidRequestException; import com.bigdata.dastor.thrift.UnavailableException; import com.bigdata.dastor.thrift.server.ThriftValidation; import com.bigdata.dastor.utils.FBUtilities; import com.bigdata.dastor.utils.LatencyTracker; import com.bigdata.dastor.utils.Pair; import com.bigdata.dastor.utils.WrappedRunnable; import com.google.common.collect.AbstractIterator; import com.google.common.collect.Multimap; public class StorageProxy implements StorageProxyMBean { private static final Logger logger = Logger.getLogger(StorageProxy.class); // BIGDATA public static final String MBEAN_OBJECT_NAME = "com.bigdata.dastor.service:type=StorageProxy"; // mbean stuff private static final LatencyTracker readStats = new LatencyTracker(); private static final LatencyTracker rangeStats = new LatencyTracker(); private static final LatencyTracker writeStats = new LatencyTracker(); private static boolean hintedHandoffEnabled = DatabaseDescriptor.hintedHandoffEnabled(); private StorageProxy() { } public static void init() // BIGDATA: use init() { MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); try { mbs.registerMBean(new StorageProxy(), new ObjectName(MBEAN_OBJECT_NAME)); // BIGDATA } catch (Exception e) { throw new RuntimeException(e); } } public static final Comparator<String> keyComparator = new Comparator<String>() { public int compare(String o1, String o2) { IPartitioner<?> p = StorageService.getPartitioner(); return p.decorateKey(o1).compareTo(p.decorateKey(o2)); } }; /** * Use this method to have these RowMutations applied * across all replicas. This method will take care * of the possibility of a replica being down and hint * the data across to some other replica. * * This is the ZERO consistency level. We do not wait for replies. * * @param mutations the mutations to be applied across the replicas */ public static void mutate(List<RowMutation> mutations) { long startTime = System.nanoTime(); try { StorageService ss = StorageService.instance; for (final RowMutation rm : mutations) { try { String table = rm.getTable(); AbstractReplicationStrategy rs = ss.getReplicationStrategy(table); List<InetAddress> naturalEndpoints = ss.getNaturalEndpoints(table, rm.key()); Multimap<InetAddress, InetAddress> hintedEndpoints = rs.getHintedEndpoints(table, naturalEndpoints); Message unhintedMessage = null; // lazy initialize for non-local, unhinted writes // 3 cases: // 1. local, unhinted write: run directly on write stage // 2. non-local, unhinted write: send row mutation message // 3. hinted write: add hint header, and send message for (Map.Entry<InetAddress, Collection<InetAddress>> entry : hintedEndpoints.asMap() .entrySet()) { InetAddress destination = entry.getKey(); Collection<InetAddress> targets = entry.getValue(); if (targets.size() == 1 && targets.iterator().next().equals(destination)) { // unhinted writes if (destination.equals(FBUtilities.getLocalAddress())) { if (logger.isDebugEnabled()) logger.debug("insert writing local key " + rm.key()); Runnable runnable = new WrappedRunnable() { public void runMayThrow() throws IOException { rm.apply(); } }; StageManager.getStage(StageManager.MUTATION_STAGE).execute(runnable); } else { if (unhintedMessage == null) unhintedMessage = rm.makeRowMutationMessage(); if (logger.isDebugEnabled()) logger.debug("insert writing key " + rm.key() + " to " + unhintedMessage.getMessageId() + "@" + destination); MessagingService.instance.sendOneWay(unhintedMessage, destination); } } else { // hinted Message hintedMessage = rm.makeRowMutationMessage(); for (InetAddress target : targets) { if (!target.equals(destination)) { addHintHeader(hintedMessage, target); if (logger.isDebugEnabled()) logger.debug("insert writing key " + rm.key() + " to " + hintedMessage.getMessageId() + "@" + destination + " for " + target); } } MessagingService.instance.sendOneWay(hintedMessage, destination); } } } catch (IOException e) { throw new RuntimeException("error inserting key " + rm.key(), e); } } } finally { writeStats.addNano(System.nanoTime() - startTime); } } private static void addHintHeader(Message message, InetAddress target) { byte[] oldHint = message.getHeader(RowMutation.HINT); byte[] hint = oldHint == null ? target.getAddress() : ArrayUtils.addAll(oldHint, target.getAddress()); message.setHeader(RowMutation.HINT, hint); } public static void mutateBlocking(List<RowMutation> mutations, ConsistencyLevel consistency_level) throws UnavailableException, TimeoutException { long startTime = System.nanoTime(); ArrayList<WriteResponseHandler> responseHandlers = new ArrayList<WriteResponseHandler>(); RowMutation mostRecentRowMutation = null; StorageService ss = StorageService.instance; try { for (RowMutation rm : mutations) { mostRecentRowMutation = rm; String table = rm.getTable(); AbstractReplicationStrategy rs = ss.getReplicationStrategy(table); List<InetAddress> naturalEndpoints = ss.getNaturalEndpoints(table, rm.key()); Collection<InetAddress> writeEndpoints = rs.getWriteEndpoints( StorageService.getPartitioner().getToken(rm.key()), table, naturalEndpoints); Multimap<InetAddress, InetAddress> hintedEndpoints = rs.getHintedEndpoints(table, writeEndpoints); int blockFor = determineBlockFor(writeEndpoints.size(), consistency_level); // avoid starting a write we know can't achieve the required consistency assureSufficientLiveNodes(blockFor, writeEndpoints, hintedEndpoints, consistency_level); // send out the writes, as in mutate() above, but this time with a callback that tracks responses final WriteResponseHandler responseHandler = ss.getWriteResponseHandler(blockFor, consistency_level, table); responseHandlers.add(responseHandler); Message unhintedMessage = null; for (Map.Entry<InetAddress, Collection<InetAddress>> entry : hintedEndpoints.asMap().entrySet()) { InetAddress destination = entry.getKey(); Collection<InetAddress> targets = entry.getValue(); if (targets.size() == 1 && targets.iterator().next().equals(destination)) { // unhinted writes if (destination.equals(FBUtilities.getLocalAddress())) { insertLocalMessage(rm, responseHandler); } else { // belongs on a different server. send it there. if (unhintedMessage == null) { unhintedMessage = rm.makeRowMutationMessage(); MessagingService.instance.addCallback(responseHandler, unhintedMessage.getMessageId()); } if (logger.isDebugEnabled()) logger.debug("insert writing key " + rm.key() + " to " + unhintedMessage.getMessageId() + "@" + destination); MessagingService.instance.sendOneWay(unhintedMessage, destination); } } else { // hinted Message hintedMessage = rm.makeRowMutationMessage(); for (InetAddress target : targets) { if (!target.equals(destination)) { addHintHeader(hintedMessage, target); if (logger.isDebugEnabled()) logger.debug("insert writing key " + rm.key() + " to " + hintedMessage.getMessageId() + "@" + destination + " for " + target); } } // (non-destination hints are part of the callback and count towards consistency only under CL.ANY) if (writeEndpoints.contains(destination) || consistency_level == ConsistencyLevel.ANY) MessagingService.instance.addCallback(responseHandler, hintedMessage.getMessageId()); MessagingService.instance.sendOneWay(hintedMessage, destination); } } } // wait for writes. throws timeoutexception if necessary for (WriteResponseHandler responseHandler : responseHandlers) { responseHandler.get(); } } catch (IOException e) { if (mostRecentRowMutation == null) throw new RuntimeException("no mutations were seen but found an error during write anyway", e); else throw new RuntimeException("error writing key " + mostRecentRowMutation.key(), e); } finally { writeStats.addNano(System.nanoTime() - startTime); } } private static void assureSufficientLiveNodes(int blockFor, Collection<InetAddress> writeEndpoints, Multimap<InetAddress, InetAddress> hintedEndpoints, ConsistencyLevel consistencyLevel) throws UnavailableException { if (consistencyLevel == ConsistencyLevel.ANY) { // ensure there are blockFor distinct living nodes (hints are ok). if (hintedEndpoints.keySet().size() < blockFor) throw new UnavailableException(); } // count destinations that are part of the desired target set int liveNodes = 0; for (InetAddress destination : hintedEndpoints.keySet()) { if (writeEndpoints.contains(destination)) liveNodes++; } if (liveNodes < blockFor) { throw new UnavailableException(); } } private static void insertLocalMessage(final RowMutation rm, final WriteResponseHandler responseHandler) { if (logger.isDebugEnabled()) logger.debug("insert writing local key " + rm.key()); Runnable runnable = new WrappedRunnable() { public void runMayThrow() throws IOException { rm.apply(); responseHandler.localResponse(); } }; StageManager.getStage(StageManager.MUTATION_STAGE).execute(runnable); } private static int determineBlockFor(int expandedTargets, ConsistencyLevel consistency_level) { switch (consistency_level) { case ONE: case ANY: return 1; case QUORUM: return (expandedTargets / 2) + 1; case DCQUORUM: case DCQUORUMSYNC: // TODO this is broken return expandedTargets; case ALL: return expandedTargets; default: throw new UnsupportedOperationException("invalid consistency level " + consistency_level); } } /** * Performs the actual reading of a row out of the StorageService, fetching * a specific set of column names from a given column family. */ public static List<Row> readProtocol(List<ReadCommand> commands, ConsistencyLevel consistency_level) throws IOException, UnavailableException, TimeoutException, InvalidRequestException { if (StorageService.instance.isBootstrapMode()) throw new InvalidRequestException("This node cannot accept reads until it has bootstrapped"); long startTime = System.nanoTime(); List<Row> rows; if (consistency_level == ConsistencyLevel.ONE) { rows = weakRead(commands); } else { assert consistency_level.getValue() >= ConsistencyLevel.QUORUM.getValue(); rows = strongRead(commands, consistency_level); } readStats.addNano(System.nanoTime() - startTime); return rows; } private static List<Row> weakRead(List<ReadCommand> commands) throws IOException, UnavailableException, TimeoutException { List<Row> rows = new ArrayList<Row>(); // send off all the commands asynchronously List<Future<Object>> localFutures = null; List<IAsyncResult> remoteResults = null; for (ReadCommand command : commands) { InetAddress endPoint = StorageService.instance.findSuitableEndPoint(command.table, command.key); if (endPoint.equals(FBUtilities.getLocalAddress())) { if (logger.isDebugEnabled()) logger.debug("weakread reading " + command + " locally"); if (localFutures == null) localFutures = new ArrayList<Future<Object>>(); Callable<Object> callable = new weakReadLocalCallable(command); localFutures.add(StageManager.getStage(StageManager.READ_STAGE).submit(callable)); } else { if (remoteResults == null) remoteResults = new ArrayList<IAsyncResult>(); Message message = command.makeReadMessage(); if (logger.isDebugEnabled()) logger.debug( "weakread reading " + command + " from " + message.getMessageId() + "@" + endPoint); if (DatabaseDescriptor.getConsistencyCheck()) message.setHeader(ReadCommand.DO_REPAIR, ReadCommand.DO_REPAIR.getBytes()); remoteResults.add(MessagingService.instance.sendRR(message, endPoint)); } } // wait for results if (localFutures != null) { for (Future<Object> future : localFutures) { Row row; try { row = (Row) future.get(); } catch (Exception e) { throw new RuntimeException(e); } rows.add(row); } } if (remoteResults != null) { for (IAsyncResult iar : remoteResults) { byte[] body; body = iar.get(DatabaseDescriptor.getRpcTimeout(), TimeUnit.MILLISECONDS); ByteArrayInputStream bufIn = new ByteArrayInputStream(body); ReadResponse response = ReadResponse.serializer().deserialize(new DataInputStream(bufIn)); if (response.row() != null) rows.add(response.row()); } } return rows; } /* * This function executes the read protocol. // 1. Get the N nodes from storage service where the data needs to be // replicated // 2. Construct a message for read\write * 3. Set one of the messages to get the data and the rest to get the digest // 4. SendRR ( to all the nodes above ) // 5. Wait for a response from at least X nodes where X <= N and the data node * 6. If the digest matches return the data. * 7. else carry out read repair by getting data from all the nodes. // 5. return success */ private static List<Row> strongRead(List<ReadCommand> commands, ConsistencyLevel consistency_level) throws IOException, UnavailableException, TimeoutException { List<QuorumResponseHandler<Row>> quorumResponseHandlers = new ArrayList<QuorumResponseHandler<Row>>(); List<InetAddress[]> commandEndPoints = new ArrayList<InetAddress[]>(); List<Row> rows = new ArrayList<Row>(); // send out read requests for (ReadCommand command : commands) { assert !command.isDigestQuery(); ReadCommand readMessageDigestOnly = command.copy(); readMessageDigestOnly.setDigestQuery(true); Message message = command.makeReadMessage(); Message messageDigestOnly = readMessageDigestOnly.makeReadMessage(); InetAddress dataPoint = StorageService.instance.findSuitableEndPoint(command.table, command.key); List<InetAddress> endpointList = StorageService.instance.getLiveNaturalEndpoints(command.table, command.key); final String table = command.table; int responseCount = determineBlockFor(DatabaseDescriptor.getReplicationFactor(table), consistency_level); if (endpointList.size() < responseCount) throw new UnavailableException(); InetAddress[] endPoints = new InetAddress[endpointList.size()]; Message messages[] = new Message[endpointList.size()]; // data-request message is sent to dataPoint, the node that will actually get // the data for us. The other replicas are only sent a digest query. int n = 0; for (InetAddress endpoint : endpointList) { Message m = endpoint.equals(dataPoint) ? message : messageDigestOnly; endPoints[n] = endpoint; messages[n++] = m; if (logger.isDebugEnabled()) logger.debug("strongread reading " + (m == message ? "data" : "digest") + " for " + command + " from " + m.getMessageId() + "@" + endpoint); } QuorumResponseHandler<Row> quorumResponseHandler = new QuorumResponseHandler<Row>(responseCount, new ReadResponseResolver(command.table, responseCount)); MessagingService.instance.sendRR(messages, endPoints, quorumResponseHandler); quorumResponseHandlers.add(quorumResponseHandler); commandEndPoints.add(endPoints); } // read results and make a second pass for any digest mismatches List<QuorumResponseHandler<Row>> repairResponseHandlers = null; for (int i = 0; i < commands.size(); i++) { QuorumResponseHandler<Row> quorumResponseHandler = quorumResponseHandlers.get(i); Row row; ReadCommand command = commands.get(i); try { long startTime2 = System.currentTimeMillis(); row = quorumResponseHandler.get(); if (row != null) rows.add(row); if (logger.isDebugEnabled()) logger.debug("quorumResponseHandler: " + (System.currentTimeMillis() - startTime2) + " ms."); } catch (DigestMismatchException ex) { if (DatabaseDescriptor.getConsistencyCheck()) { if (logger.isDebugEnabled()) logger.debug("Digest mismatch:", ex); int responseCount = determineBlockFor(DatabaseDescriptor.getReplicationFactor(command.table), consistency_level); QuorumResponseHandler<Row> qrhRepair = new QuorumResponseHandler<Row>(responseCount, new ReadResponseResolver(command.table, responseCount)); Message messageRepair = command.makeReadMessage(); MessagingService.instance.sendRR(messageRepair, commandEndPoints.get(i), qrhRepair); if (repairResponseHandlers == null) repairResponseHandlers = new ArrayList<QuorumResponseHandler<Row>>(); repairResponseHandlers.add(qrhRepair); } } } // read the results for the digest mismatch retries if (repairResponseHandlers != null) { for (QuorumResponseHandler<Row> handler : repairResponseHandlers) { try { Row row = handler.get(); if (row != null) rows.add(row); } catch (DigestMismatchException e) { throw new AssertionError(e); // full data requested from each node here, no digests should be sent } } } return rows; } /* * This function executes the read protocol locally. Consistency checks are performed in the background. */ public static List<Row> getRangeSlice(RangeSliceCommand command, ConsistencyLevel consistency_level) throws IOException, UnavailableException, TimeoutException { if (logger.isDebugEnabled()) logger.debug(command); long startTime = System.nanoTime(); final String table = command.keyspace; int responseCount = determineBlockFor(DatabaseDescriptor.getReplicationFactor(table), consistency_level); List<AbstractBounds> ranges = getRestrictedRanges(command.range); // now scan until we have enough results List<Row> rows = new ArrayList<Row>(command.max_keys); for (AbstractBounds range : ranges) { List<InetAddress> liveEndpoints = StorageService.instance.getLiveNaturalEndpoints(command.keyspace, range.right); if (liveEndpoints.size() < responseCount) throw new UnavailableException(); DatabaseDescriptor.getEndPointSnitch(command.keyspace).sortByProximity(FBUtilities.getLocalAddress(), liveEndpoints); List<InetAddress> endpoints = liveEndpoints.subList(0, responseCount); RangeSliceCommand c2 = new RangeSliceCommand(command.keyspace, command.column_family, command.super_column, command.predicate, range, command.max_keys); Message message = c2.getMessage(); // collect replies and resolve according to consistency level RangeSliceResponseResolver resolver = new RangeSliceResponseResolver(command.keyspace, endpoints, StorageService.getPartitioner()); QuorumResponseHandler<List<Row>> handler = new QuorumResponseHandler<List<Row>>(responseCount, resolver); for (InetAddress endpoint : endpoints) { MessagingService.instance.sendRR(message, endpoint, handler); if (logger.isDebugEnabled()) logger.debug("reading " + c2 + " from " + message.getMessageId() + "@" + endpoint); } // TODO read repair on remaining replicas? // if we're done, great, otherwise, move to the next range try { if (logger.isDebugEnabled()) { for (Row row : handler.get()) { logger.debug("range slices read " + row.key); } } rows.addAll(handler.get()); } catch (DigestMismatchException e) { throw new AssertionError(e); // no digests in range slices yet } if (rows.size() >= command.max_keys) break; } rangeStats.addNano(System.nanoTime() - startTime); return rows.size() > command.max_keys ? rows.subList(0, command.max_keys) : rows; } /** * Compute all ranges we're going to query, in sorted order. Nodes can be replica destinations for many ranges, * so we need to restrict each scan to the specific range we want, or else we'd get duplicate results. */ static List<AbstractBounds> getRestrictedRanges(final AbstractBounds queryRange) { // special case for bounds containing exactly 1 (non-minimum) token if (queryRange instanceof Bounds && queryRange.left.equals(queryRange.right) && !queryRange.left.equals(StorageService.getPartitioner().getMinimumToken())) { if (logger.isDebugEnabled()) logger.debug("restricted single token match for query " + queryRange); return Collections.singletonList(queryRange); } TokenMetadata tokenMetadata = StorageService.instance.getTokenMetadata(); List<AbstractBounds> ranges = new ArrayList<AbstractBounds>(); // divide the queryRange into pieces delimited by the ring and minimum tokens Iterator<Token> ringIter = TokenMetadata.ringIterator(tokenMetadata.sortedTokens(), queryRange.left, true); AbstractBounds remainder = queryRange; while (ringIter.hasNext()) { Token token = ringIter.next(); if (remainder == null || !remainder.contains(token)) // no more splits break; Pair<AbstractBounds, AbstractBounds> splits = remainder.split(token); ranges.add(splits.left); remainder = splits.right; } if (remainder != null) ranges.add(remainder); if (logger.isDebugEnabled()) logger.debug("restricted ranges for query " + queryRange + " are " + ranges); return ranges; } public long getReadOperations() { return readStats.getOpCount(); } public long getTotalReadLatencyMicros() { return readStats.getTotalLatencyMicros(); } public double getRecentReadLatencyMicros() { return readStats.getRecentLatencyMicros(); } // BIGDATA @Override public double getRecentReadLatencyMs() { return getRecentReadLatencyMicros() / 1000; } // BIGDATA @Override public double getRecentReadThroughput() { return readStats.getRecentThroughput(); } public long getRangeOperations() { return rangeStats.getOpCount(); } public long getTotalRangeLatencyMicros() { return rangeStats.getTotalLatencyMicros(); } public double getRecentRangeLatencyMicros() { return rangeStats.getRecentLatencyMicros(); } // BIGDATA @Override public double getRecentRangeThroughput() { return rangeStats.getRecentThroughput(); } // BIGDATA @Override public double getRecentRangeLatencyMs() { return getRecentRangeLatencyMicros() / 1000; } public long getWriteOperations() { return writeStats.getOpCount(); } public long getTotalWriteLatencyMicros() { return writeStats.getTotalLatencyMicros(); } public double getRecentWriteLatencyMicros() { return writeStats.getRecentLatencyMicros(); } // BIGDATA @Override public double getRecentWriteLatencyMs() { return getRecentWriteLatencyMicros() / 1000; } // BIGDATA @Override public double getRecentWriteThroughput() { return writeStats.getRecentThroughput(); } public boolean getHintedHandoffEnabled() { return hintedHandoffEnabled; } public void setHintedHandoffEnabled(boolean b) { hintedHandoffEnabled = b; } public static boolean isHintedHandoffEnabled() { return hintedHandoffEnabled; } // BIGDATA @Override public void graceResetClusterCF(String tableName, String columnFamily) throws InvalidRequestException { if (DatabaseDescriptor.isSystemTable(tableName)) { throw new InvalidRequestException(tableName + " not allow for the operation."); } String realCfName = ThriftValidation.validateUdBucketName(tableName, columnFamily, false); ThriftValidation.validateColumnFamily(tableName, realCfName); try { Collector.setCFCResetOp(tableName, realCfName, columnFamily); } catch (UnavailableException e) { InvalidRequestException ire = new InvalidRequestException(); ire.initCause(e); throw ire; } catch (TimeoutException e) { InvalidRequestException ire = new InvalidRequestException(); ire.initCause(e); throw ire; } } // BIGDATA @Override public void graceResetClusterCFUndo(String tableName, String columnFamily) throws InvalidRequestException { if (DatabaseDescriptor.isSystemTable(tableName)) { throw new InvalidRequestException(tableName + " not allow for the operation."); } String realCfName = ThriftValidation.validateUdBucketName(tableName, columnFamily, false); ThriftValidation.validateColumnFamily(tableName, realCfName); try { Collector.deleteCFCResetOp(tableName, realCfName, columnFamily); } catch (UnavailableException e) { InvalidRequestException ire = new InvalidRequestException(); ire.initCause(e); throw ire; } catch (TimeoutException e) { InvalidRequestException ire = new InvalidRequestException(); ire.initCause(e); throw ire; } catch (IOException e) { InvalidRequestException ire = new InvalidRequestException(); ire.initCause(e); throw ire; } } static class weakReadLocalCallable implements Callable<Object> { private ReadCommand command; weakReadLocalCallable(ReadCommand command) { this.command = command; } public Object call() throws IOException { if (logger.isDebugEnabled()) logger.debug("weakreadlocal reading " + command); Table table = Table.open(command.table); Row row = command.getRow(table); // Do the consistency checks in the background if (DatabaseDescriptor.getConsistencyCheck()) { List<InetAddress> endpoints = StorageService.instance.getLiveNaturalEndpoints(command.table, command.key); if (endpoints.size() > 1) StorageService.instance.doConsistencyCheck(row, endpoints, command); } return row; } } }