Java tutorial
/* * Copyright 2008-2013 LinkedIn, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of * the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. */ package voldemort; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.BufferedWriter; import java.io.ByteArrayInputStream; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.EOFException; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FileWriter; import java.io.FilterOutputStream; import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.PrintStream; import java.io.StringReader; import java.io.StringWriter; import java.io.Writer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Properties; import java.util.Scanner; import java.util.Set; import joptsimple.OptionParser; import joptsimple.OptionSet; import org.apache.avro.Schema; import org.apache.avro.generic.GenericDatumReader; import org.apache.avro.generic.GenericRecord; import org.apache.avro.io.JsonDecoder; import org.apache.commons.codec.DecoderException; import org.apache.commons.io.FileUtils; import org.codehaus.jackson.JsonFactory; import org.codehaus.jackson.JsonGenerator; import org.codehaus.jackson.map.ObjectMapper; import voldemort.client.ClientConfig; import voldemort.client.protocol.admin.AdminClient; import voldemort.client.protocol.admin.AdminClientConfig; import voldemort.client.protocol.admin.QueryKeyResult; import voldemort.cluster.Cluster; import voldemort.cluster.Node; import voldemort.cluster.Zone; import voldemort.routing.BaseStoreRoutingPlan; import voldemort.routing.StoreRoutingPlan; import voldemort.serialization.DefaultSerializerFactory; import voldemort.serialization.SerializationException; import voldemort.serialization.Serializer; import voldemort.serialization.SerializerDefinition; import voldemort.serialization.SerializerFactory; import voldemort.serialization.StringSerializer; import voldemort.serialization.avro.versioned.SchemaEvolutionValidator; import voldemort.serialization.json.JsonReader; import voldemort.server.rebalance.RebalancerState; import voldemort.store.StoreDefinition; import voldemort.store.compress.CompressionStrategy; import voldemort.store.compress.CompressionStrategyFactory; import voldemort.store.metadata.MetadataStore; import voldemort.store.metadata.MetadataStore.VoldemortState; import voldemort.store.quota.QuotaUtils; import voldemort.store.readonly.ReadOnlyStorageConfiguration; import voldemort.store.system.SystemStoreConstants; import voldemort.utils.ByteArray; import voldemort.utils.ByteUtils; import voldemort.utils.CmdUtils; import voldemort.utils.MetadataVersionStoreUtils; import voldemort.utils.Pair; import voldemort.utils.StoreDefinitionUtils; import voldemort.utils.Utils; import voldemort.versioning.VectorClock; import voldemort.versioning.Versioned; import voldemort.xml.ClusterMapper; import voldemort.xml.StoreDefinitionsMapper; import com.google.common.base.Joiner; import com.google.common.collect.AbstractIterator; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.sleepycat.persist.StoreNotFoundException; /** * Provides a command line interface to the * {@link voldemort.client.protocol.admin.AdminClient} */ public class VoldemortAdminTool { private static final String ALL_METADATA = "all"; @SuppressWarnings("unchecked") public static void main(String[] args) throws Exception { OptionParser parser = new OptionParser(); // This is a generic argument that should be eventually supported by all // RW operations. // If you omit this argument the operation will be executed in a "batch" // mode which is useful for scripting // Otherwise you will be presented with a summary of changes and with a // Y/N prompt parser.accepts("auto", "[OPTIONAL] enable auto/batch mode"); parser.accepts("help", "print help information"); parser.accepts("url", "[REQUIRED] bootstrap URL").withRequiredArg().describedAs("bootstrap-url") .ofType(String.class); parser.accepts("node", "node id").withRequiredArg().describedAs("node-id").ofType(Integer.class); parser.accepts("delete-partitions", "Delete partitions").withRequiredArg().describedAs("partition-ids") .withValuesSeparatedBy(',').ofType(Integer.class); parser.accepts("restore", "Restore from replication [ Optional parallelism param - Default - 5 ]") .withOptionalArg().describedAs("parallelism").ofType(Integer.class); parser.accepts("ascii", "Fetch keys as ASCII"); parser.accepts("fetch-keys", "Fetch keys").withOptionalArg().describedAs("partition-ids") .withValuesSeparatedBy(',').ofType(Integer.class); parser.accepts("fetch-entries", "Fetch full entries").withOptionalArg().describedAs("partition-ids") .withValuesSeparatedBy(',').ofType(Integer.class); parser.accepts("outdir", "Output directory").withRequiredArg().describedAs("output-directory") .ofType(String.class); parser.accepts("nodes", "list of nodes").withRequiredArg().describedAs("nodes").withValuesSeparatedBy(',') .ofType(Integer.class); parser.accepts("stores", "Store names").withRequiredArg().describedAs("store-names") .withValuesSeparatedBy(',').ofType(String.class); parser.accepts("store", "Store name for querying keys").withRequiredArg().describedAs("store-name") .ofType(String.class); parser.accepts("add-stores", "Add stores in this stores.xml").withRequiredArg() .describedAs("stores.xml containing just the new stores").ofType(String.class); parser.accepts("delete-store", "Delete store").withRequiredArg().describedAs("store-name") .ofType(String.class); parser.accepts("update-entries", "Insert or update entries").withRequiredArg() .describedAs("input-directory").ofType(String.class); parser.accepts("get-metadata", "retreive metadata information " + MetadataStore.METADATA_KEYS) .withOptionalArg().describedAs("metadata-key").ofType(String.class); parser.accepts("check-metadata", "retreive metadata information from all nodes and checks if they are consistent across [ " + MetadataStore.CLUSTER_KEY + " | " + MetadataStore.STORES_KEY + " | " + MetadataStore.REBALANCING_SOURCE_CLUSTER_XML + " | " + MetadataStore.SERVER_STATE_KEY + " ]") .withRequiredArg().describedAs("metadata-key").ofType(String.class); parser.accepts("ro-metadata", "retrieve version information [current | max | storage-format]") .withRequiredArg().describedAs("type").ofType(String.class); parser.accepts("truncate", "truncate a store").withRequiredArg().describedAs("store-name") .ofType(String.class); parser.accepts("set-metadata", "Forceful setting of metadata [ " + MetadataStore.CLUSTER_KEY + " | " + MetadataStore.STORES_KEY + " | " + MetadataStore.SERVER_STATE_KEY + " | " + MetadataStore.REBALANCING_SOURCE_CLUSTER_XML + " | " + MetadataStore.REBALANCING_STEAL_INFO + " ]") .withRequiredArg().describedAs("metadata-key").ofType(String.class); parser.accepts("set-metadata-value", "The value for the set-metadata [ " + MetadataStore.CLUSTER_KEY + " | " + MetadataStore.STORES_KEY + ", " + MetadataStore.REBALANCING_SOURCE_CLUSTER_XML + ", " + MetadataStore.REBALANCING_STEAL_INFO + " ] - xml file location, [ " + MetadataStore.SERVER_STATE_KEY + " ] - " + MetadataStore.VoldemortState.NORMAL_SERVER + "," + MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER) .withRequiredArg().describedAs("metadata-value").ofType(String.class); parser.accepts("set-metadata-pair", "Atomic setting of metadata pair [ " + MetadataStore.CLUSTER_KEY + " & " + MetadataStore.STORES_KEY + " ]") .withRequiredArg().describedAs("metadata-keys-pair").withValuesSeparatedBy(',') .ofType(String.class); parser.accepts("set-metadata-value-pair", "The value for the set-metadata pair [ " + MetadataStore.CLUSTER_KEY + " & " + MetadataStore.STORES_KEY + " ]") .withRequiredArg().describedAs("metadata-value-pair").withValuesSeparatedBy(',') .ofType(String.class); parser.accepts("clear-rebalancing-metadata", "Remove the metadata related to rebalancing"); parser.accepts("async", "a) Get a list of async job ids [get] b) Stop async job ids [stop] ") .withRequiredArg().describedAs("op-type").ofType(String.class); parser.accepts("async-id", "Comma separated list of async ids to stop").withOptionalArg() .describedAs("job-ids").withValuesSeparatedBy(',').ofType(Integer.class); parser.accepts("repair-job", "Clean after rebalancing is done"); parser.accepts("prune-job", "Prune versioned put data, after rebalancing"); parser.accepts("purge-slops", "Purge the slop stores selectively, based on nodeId or zoneId"); parser.accepts("native-backup", "Perform a native backup").withRequiredArg().describedAs("store-name") .ofType(String.class); parser.accepts("backup-dir").withRequiredArg().describedAs("backup-directory").ofType(String.class); parser.accepts("backup-timeout").withRequiredArg() .describedAs("minutes to wait for backup completion, default 30 mins").ofType(Integer.class); parser.accepts("backup-verify", "If provided, backup will also verify checksum (with extra overhead)"); parser.accepts("backup-incremental", "Perform an incremental backup for point-in-time recovery." + " By default backup has latest consistent snapshot."); parser.accepts("zone", "zone id").withRequiredArg().describedAs("zone-id").ofType(Integer.class); parser.accepts("rollback", "rollback a store").withRequiredArg().describedAs("store-name") .ofType(String.class); parser.accepts("version", "Push version of store to rollback to").withRequiredArg().describedAs("version") .ofType(Long.class); parser.accepts("verify-metadata-version", "Verify the version of Metadata on all the cluster nodes"); parser.accepts("synchronize-metadata-version", "Synchronize the metadata versions across all the nodes."); parser.accepts("reserve-memory", "Memory in MB to reserve for the store").withRequiredArg() .describedAs("size-in-mb").ofType(Long.class); parser.accepts("query-key", "Get values of a key on specific node").withRequiredArg() .describedAs("query-key").ofType(String.class); parser.accepts("query-key-format", "Format of the query key. Can be one of [hex|readable]") .withRequiredArg().describedAs("key-format").ofType(String.class); parser.accepts("show-routing-plan", "Routing plan of the specified keys").withRequiredArg() .describedAs("keys-to-be-routed").withValuesSeparatedBy(',').ofType(String.class); parser.accepts("mirror-from-url", "Cluster url to mirror data from").withRequiredArg() .describedAs("mirror-cluster-bootstrap-url").ofType(String.class); parser.accepts("mirror-node", "Node id in the mirror cluster to mirror from").withRequiredArg() .describedAs("id-of-mirror-node").ofType(Integer.class); parser.accepts("fetch-orphaned", "Fetch any orphaned keys/entries in the node"); parser.accepts("set-quota", "Enforce some quota on the servers").withRequiredArg().describedAs("quota-type") .ofType(String.class); parser.accepts("quota-value", "Value of the quota enforced on the servers").withRequiredArg() .describedAs("quota-value").ofType(String.class); parser.accepts("unset-quota", "Remove some quota already enforced on the servers").withRequiredArg() .describedAs("quota-type").ofType(String.class); // TODO add a way to retrieve all quotas for a given store. parser.accepts("get-quota", "Retrieve some quota already enforced on the servers").withRequiredArg() .describedAs("quota-type").ofType(String.class); OptionSet options = parser.parse(args); if (options.has("help")) { printHelp(System.out, parser); System.exit(0); } Set<String> missing = CmdUtils.missing(options, "url", "node"); if (missing.size() > 0) { // Not the most elegant way to do this // basically check if only "node" is missing for these set of // options; all these can live without explicit node ids if (!(missing.equals(ImmutableSet.of("node")) && (options.has("add-stores") || options.has("delete-store") || options.has("ro-metadata") || options.has("set-metadata") || options.has("set-metadata-pair") || options.has("get-metadata") || options.has("check-metadata")) || options.has("truncate") || options.has("clear-rebalancing-metadata") || options.has("async") || options.has("native-backup") || options.has("rollback") || options.has("verify-metadata-version") || options.has("reserve-memory") || options.has("purge-slops") || options.has("show-routing-plan") || options.has("query-key") || options.has("set-quota") || options.has("unset-quota") || options.has("get-quota"))) { System.err.println("Missing required arguments: " + Joiner.on(", ").join(missing)); printHelp(System.err, parser); System.exit(1); } } try { String url = (String) options.valueOf("url"); Integer nodeId = CmdUtils.valueOf(options, "node", -1); int parallelism = CmdUtils.valueOf(options, "restore", 5); Integer zoneId = CmdUtils.valueOf(options, "zone", -1); AdminClient adminClient = new AdminClient(url, new AdminClientConfig(), new ClientConfig()); List<String> storeNames = null; if (options.has("store") && options.has("stores")) { throw new VoldemortException("Must not specify both --stores and --store options"); } else if (options.has("stores")) { storeNames = (List<String>) options.valuesOf("stores"); } else if (options.has("store")) { storeNames = Arrays.asList((String) options.valueOf("store")); } String outputDir = null; if (options.has("outdir")) { outputDir = (String) options.valueOf("outdir"); } if (options.has("add-stores")) { String storesXml = (String) options.valueOf("add-stores"); executeAddStores(adminClient, storesXml, nodeId); } else if (options.has("async")) { String asyncKey = (String) options.valueOf("async"); List<Integer> asyncIds = null; if (options.hasArgument("async-id")) asyncIds = (List<Integer>) options.valuesOf("async-id"); executeAsync(nodeId, adminClient, asyncKey, asyncIds); } else if (options.has("check-metadata")) { String metadataKey = (String) options.valueOf("check-metadata"); executeCheckMetadata(adminClient, metadataKey); } else if (options.has("delete-partitions")) { System.out.println("Starting delete-partitions"); List<Integer> partitionIdList = (List<Integer>) options.valuesOf("delete-partitions"); executeDeletePartitions(nodeId, adminClient, partitionIdList, storeNames); System.out.println("Finished delete-partitions"); } else if (options.has("ro-metadata")) { String type = (String) options.valueOf("ro-metadata"); executeROMetadata(nodeId, adminClient, storeNames, type); } else if (options.has("reserve-memory")) { if (!options.has("stores")) { Utils.croak("Specify the list of stores to reserve memory"); } long reserveMB = (Long) options.valueOf("reserve-memory"); adminClient.quotaMgmtOps.reserveMemory(nodeId, storeNames, reserveMB); } else if (options.has("get-metadata")) { String metadataKey = ALL_METADATA; if (options.hasArgument("get-metadata")) { metadataKey = (String) options.valueOf("get-metadata"); } executeGetMetadata(nodeId, adminClient, metadataKey, outputDir); } else if (options.has("mirror-from-url")) { if (!options.has("mirror-node")) { Utils.croak("Specify the mirror node to fetch from"); } if (nodeId == -1) { System.err.println("Cannot run mirroring without node id"); System.exit(1); } Integer mirrorNodeId = CmdUtils.valueOf(options, "mirror-node", -1); if (mirrorNodeId == -1) { System.err.println("Cannot run mirroring without mirror node id"); System.exit(1); } adminClient.restoreOps.mirrorData(nodeId, mirrorNodeId, (String) options.valueOf("mirror-from-url"), storeNames); } else if (options.has("clear-rebalancing-metadata")) { executeClearRebalancing(nodeId, adminClient); } else if (options.has("prune-job")) { if (storeNames == null) { Utils.croak("Must specify --stores to run the prune job"); } executePruneJob(nodeId, adminClient, storeNames); } else if (options.has("fetch-keys")) { boolean useAscii = options.has("ascii"); System.out.println("Starting fetch keys"); List<Integer> partitionIdList = null; if (options.hasArgument("fetch-keys")) partitionIdList = (List<Integer>) options.valuesOf("fetch-keys"); executeFetchKeys(nodeId, adminClient, partitionIdList, outputDir, storeNames, useAscii, options.has("fetch-orphaned")); } else if (options.has("repair-job")) { executeRepairJob(nodeId, adminClient); } else if (options.has("set-metadata-pair")) { List<String> metadataKeyPair = (List<String>) options.valuesOf("set-metadata-pair"); if (metadataKeyPair.size() != 2) { throw new VoldemortException( "Missing set-metadata-pair keys (only two keys are needed and allowed)"); } if (!options.has("set-metadata-value-pair")) { throw new VoldemortException("Missing set-metadata-value-pair"); } else { List<String> metadataValuePair = (List<String>) options.valuesOf("set-metadata-value-pair"); if (metadataValuePair.size() != 2) { throw new VoldemortException( "Missing set-metadata--value-pair values (only two values are needed and allowed)"); } if (metadataKeyPair.contains(MetadataStore.CLUSTER_KEY) && metadataKeyPair.contains(MetadataStore.STORES_KEY)) { ClusterMapper clusterMapper = new ClusterMapper(); StoreDefinitionsMapper storeDefsMapper = new StoreDefinitionsMapper(); // original metadata Integer nodeIdToGetStoreXMLFrom = nodeId; if (nodeId < 0) { Collection<Node> nodes = adminClient.getAdminClientCluster().getNodes(); if (nodes.isEmpty()) { throw new VoldemortException("No nodes in this cluster"); } else { nodeIdToGetStoreXMLFrom = nodes.iterator().next().getId(); } } Versioned<String> storesXML = adminClient.metadataMgmtOps .getRemoteMetadata(nodeIdToGetStoreXMLFrom, MetadataStore.STORES_KEY); List<StoreDefinition> oldStoreDefs = storeDefsMapper .readStoreList(new StringReader(storesXML.getValue())); String clusterXMLPath = metadataValuePair .get(metadataKeyPair.indexOf(MetadataStore.CLUSTER_KEY)); clusterXMLPath = clusterXMLPath.replace("~", System.getProperty("user.home")); if (!Utils.isReadableFile(clusterXMLPath)) throw new VoldemortException("Cluster xml file path incorrect"); Cluster cluster = clusterMapper.readCluster(new File(clusterXMLPath)); String storesXMLPath = metadataValuePair .get(metadataKeyPair.indexOf(MetadataStore.STORES_KEY)); storesXMLPath = storesXMLPath.replace("~", System.getProperty("user.home")); if (!Utils.isReadableFile(storesXMLPath)) throw new VoldemortException("Stores definition xml file path incorrect"); List<StoreDefinition> newStoreDefs = storeDefsMapper.readStoreList(new File(storesXMLPath)); checkSchemaCompatibility(newStoreDefs); executeSetMetadataPair(nodeId, adminClient, MetadataStore.CLUSTER_KEY, clusterMapper.writeCluster(cluster), MetadataStore.STORES_KEY, storeDefsMapper.writeStoreList(newStoreDefs)); executeUpdateMetadataVersionsOnStores(adminClient, oldStoreDefs, newStoreDefs); } else { throw new VoldemortException("set-metadata-pair keys should be <cluster.xml, stores.xml>"); } } } else if (options.has("set-metadata")) { String metadataKey = (String) options.valueOf("set-metadata"); if (!options.has("set-metadata-value")) { throw new VoldemortException("Missing set-metadata-value"); } else { String metadataValue = (String) options.valueOf("set-metadata-value"); if (metadataKey.compareTo(MetadataStore.CLUSTER_KEY) == 0 || metadataKey.compareTo(MetadataStore.REBALANCING_SOURCE_CLUSTER_XML) == 0) { if (!Utils.isReadableFile(metadataValue)) throw new VoldemortException("Cluster xml file path incorrect"); ClusterMapper mapper = new ClusterMapper(); Cluster newCluster = mapper.readCluster(new File(metadataValue)); if (options.has("auto")) { executeSetMetadata(nodeId, adminClient, metadataKey, mapper.writeCluster(newCluster)); } else { if (confirmMetadataUpdate(nodeId, adminClient, mapper.writeCluster(newCluster))) { executeSetMetadata(nodeId, adminClient, metadataKey, mapper.writeCluster(newCluster)); } else { System.out.println("New metadata has not been set"); } } } else if (metadataKey.compareTo(MetadataStore.SERVER_STATE_KEY) == 0) { VoldemortState newState = VoldemortState.valueOf(metadataValue); executeSetMetadata(nodeId, adminClient, MetadataStore.SERVER_STATE_KEY, newState.toString()); } else if (metadataKey.compareTo(MetadataStore.STORES_KEY) == 0) { if (!Utils.isReadableFile(metadataValue)) throw new VoldemortException("Stores definition xml file path incorrect"); StoreDefinitionsMapper mapper = new StoreDefinitionsMapper(); List<StoreDefinition> newStoreDefs = mapper.readStoreList(new File(metadataValue)); checkSchemaCompatibility(newStoreDefs); // original metadata Integer nodeIdToGetStoreXMLFrom = nodeId; if (nodeId < 0) { Collection<Node> nodes = adminClient.getAdminClientCluster().getNodes(); if (nodes.isEmpty()) { throw new VoldemortException("No nodes in this cluster"); } else { nodeIdToGetStoreXMLFrom = nodes.iterator().next().getId(); } } Versioned<String> storesXML = adminClient.metadataMgmtOps .getRemoteMetadata(nodeIdToGetStoreXMLFrom, MetadataStore.STORES_KEY); List<StoreDefinition> oldStoreDefs = mapper .readStoreList(new StringReader(storesXML.getValue())); if (options.has("auto")) { executeSetMetadata(nodeId, adminClient, MetadataStore.STORES_KEY, mapper.writeStoreList(newStoreDefs)); executeUpdateMetadataVersionsOnStores(adminClient, oldStoreDefs, newStoreDefs); } else { if (confirmMetadataUpdate(nodeId, adminClient, storesXML.getValue())) { executeSetMetadata(nodeId, adminClient, MetadataStore.STORES_KEY, mapper.writeStoreList(newStoreDefs)); if (nodeId >= 0) { System.err.println( "WARNING: Metadata version update of stores goes to all servers, " + "although this set-metadata oprations only goes to node " + nodeId); } executeUpdateMetadataVersionsOnStores(adminClient, oldStoreDefs, newStoreDefs); } else { System.out.println("New metadata has not been set"); } } } else if (metadataKey.compareTo(MetadataStore.REBALANCING_STEAL_INFO) == 0) { if (!Utils.isReadableFile(metadataValue)) throw new VoldemortException("Rebalancing steal info file path incorrect"); String rebalancingStealInfoJsonString = FileUtils.readFileToString(new File(metadataValue)); RebalancerState state = RebalancerState.create(rebalancingStealInfoJsonString); executeSetMetadata(nodeId, adminClient, MetadataStore.REBALANCING_STEAL_INFO, state.toJsonString()); } else { throw new VoldemortException("Incorrect metadata key"); } } } else if (options.has("native-backup")) { if (!options.has("backup-dir")) { Utils.croak("A backup directory must be specified with backup-dir option"); } String backupDir = (String) options.valueOf("backup-dir"); String storeName = (String) options.valueOf("native-backup"); int timeout = CmdUtils.valueOf(options, "backup-timeout", 30); adminClient.storeMntOps.nativeBackup(nodeId, storeName, backupDir, timeout, options.has("backup-verify"), options.has("backup-incremental")); } else if (options.has("rollback")) { if (!options.has("version")) { Utils.croak("A read-only push version must be specified with rollback option"); } String storeName = (String) options.valueOf("rollback"); long pushVersion = (Long) options.valueOf("version"); executeRollback(nodeId, storeName, pushVersion, adminClient); } else if (options.has("query-key")) { String key = (String) options.valueOf("query-key"); String keyFormat = (String) options.valueOf("query-key-format"); if (keyFormat == null) { keyFormat = "hex"; } if (!keyFormat.equals("hex") && !keyFormat.equals("readable")) { throw new VoldemortException("--query-key-format must be hex or readable"); } executeQueryKey(nodeId, adminClient, storeNames, key, keyFormat); } else if (options.has("restore")) { if (nodeId == -1) { System.err.println("Cannot run restore without node id"); System.exit(1); } System.out.println("Starting restore"); adminClient.restoreOps.restoreDataFromReplications(nodeId, parallelism, zoneId); System.out.println("Finished restore"); } else if (options.has("delete-store")) { String storeName = (String) options.valueOf("delete-store"); executeDeleteStore(adminClient, storeName, nodeId); } else if (options.has("truncate")) { String storeName = (String) options.valueOf("truncate"); executeTruncateStore(nodeId, adminClient, storeName); } else if (options.has("update-entries")) { String inputDir = (String) options.valueOf("update-entries"); executeUpdateEntries(nodeId, adminClient, storeNames, inputDir); } else if (options.has("fetch-entries")) { boolean useAscii = options.has("ascii"); System.out.println("Starting fetch entries"); List<Integer> partitionIdList = null; if (options.hasArgument("fetch-entries")) partitionIdList = (List<Integer>) options.valuesOf("fetch-entries"); executeFetchEntries(nodeId, adminClient, partitionIdList, outputDir, storeNames, useAscii, options.has("fetch-orphaned")); } else if (options.has("purge-slops")) { List<Integer> nodesToPurge = null; if (options.has("nodes")) { nodesToPurge = (List<Integer>) options.valuesOf("nodes"); } if (nodesToPurge == null && zoneId == -1 && storeNames == null) { Utils.croak("Must specify atleast one of --nodes, --zone-id or --stores with --purge-slops"); } executePurgeSlops(adminClient, nodesToPurge, zoneId, storeNames); } else if (options.has("synchronize-metadata-version")) { synchronizeMetadataVersion(adminClient, nodeId); } else if (options.has("verify-metadata-version")) { checkMetadataVersion(adminClient); } else if (options.has("show-routing-plan")) { if (!options.has("store")) { Utils.croak("Must specify the store the keys belong to using --store "); } String storeName = (String) options.valueOf("store"); List<String> keysToRoute = (List<String>) options.valuesOf("show-routing-plan"); if (keysToRoute == null || keysToRoute.size() == 0) { Utils.croak("Must specify comma separated keys list in hex format"); } executeShowRoutingPlan(adminClient, storeName, keysToRoute); } else if (options.has("set-quota")) { String quotaType = (String) options.valueOf("set-quota"); Set<String> validQuotaTypes = QuotaUtils.validQuotaTypes(); if (!validQuotaTypes.contains(quotaType)) { Utils.croak("Specify a valid quota type from :" + validQuotaTypes); } if (!options.has("store")) { Utils.croak("Must specify the store to enforce the quota on. "); } if (!options.has("quota-value")) { Utils.croak("Must specify the value of the quota being set"); } String storeName = (String) options.valueOf("store"); String quotaValue = (String) options.valueOf("quota-value"); executeSetQuota(adminClient, storeName, quotaType, quotaValue); } else if (options.has("unset-quota")) { String quotaType = (String) options.valueOf("unset-quota"); Set<String> validQuotaTypes = QuotaUtils.validQuotaTypes(); if (!validQuotaTypes.contains(quotaType)) { Utils.croak("Specify a valid quota type from :" + validQuotaTypes); } if (!options.has("store")) { Utils.croak("Must specify the store to enforce the quota on. "); } String storeName = (String) options.valueOf("store"); executeUnsetQuota(adminClient, storeName, quotaType); } else if (options.has("get-quota")) { String quotaType = (String) options.valueOf("get-quota"); Set<String> validQuotaTypes = QuotaUtils.validQuotaTypes(); if (!validQuotaTypes.contains(quotaType)) { Utils.croak("Specify a valid quota type from :" + validQuotaTypes); } if (!options.has("store")) { Utils.croak("Must specify the store to enforce the quota on. "); } String storeName = (String) options.valueOf("store"); executeGetQuota(adminClient, storeName, quotaType); } else { Utils.croak("At least one of (delete-partitions, restore, add-node, fetch-entries, " + "fetch-keys, add-stores, delete-store, update-entries, get-metadata, ro-metadata, " + "set-metadata, check-metadata, clear-rebalancing-metadata, async, " + "repair-job, native-backup, rollback, reserve-memory, mirror-url," + " verify-metadata-version, prune-job, purge-slops) must be specified"); } } catch (Exception e) { e.printStackTrace(); Utils.croak(e.getMessage()); } } private static void checkSchemaCompatibility(List<StoreDefinition> storeDefs) { String AVRO_GENERIC_VERSIONED_TYPE_NAME = "avro-generic-versioned"; for (StoreDefinition storeDef : storeDefs) { SerializerDefinition keySerDef = storeDef.getKeySerializer(); SerializerDefinition valueSerDef = storeDef.getValueSerializer(); if (keySerDef.getName().equals(AVRO_GENERIC_VERSIONED_TYPE_NAME)) { SchemaEvolutionValidator.checkSchemaCompatibility(keySerDef); } if (valueSerDef.getName().equals(AVRO_GENERIC_VERSIONED_TYPE_NAME)) { SchemaEvolutionValidator.checkSchemaCompatibility(valueSerDef); } } } private static String getMetadataVersionsForNode(AdminClient adminClient, int nodeId) { List<Integer> partitionIdList = Lists.newArrayList(); for (Node node : adminClient.getAdminClientCluster().getNodes()) { partitionIdList.addAll(node.getPartitionIds()); } Iterator<Pair<ByteArray, Versioned<byte[]>>> entriesIterator = adminClient.bulkFetchOps.fetchEntries(nodeId, SystemStoreConstants.SystemStoreName.voldsys$_metadata_version_persistence.name(), partitionIdList, null, true); Serializer<String> serializer = new StringSerializer("UTF8"); String keyObject = null; String valueObject = null; while (entriesIterator.hasNext()) { try { Pair<ByteArray, Versioned<byte[]>> kvPair = entriesIterator.next(); byte[] keyBytes = kvPair.getFirst().get(); byte[] valueBytes = kvPair.getSecond().getValue(); keyObject = serializer.toObject(keyBytes); if (!keyObject.equals(MetadataVersionStoreUtils.VERSIONS_METADATA_KEY)) { continue; } valueObject = serializer.toObject(valueBytes); } catch (Exception e) { System.err.println( "Error while retrieving Metadata versions from node : " + nodeId + ". Exception = \n"); e.printStackTrace(); System.exit(-1); } } return valueObject; } private static void checkMetadataVersion(AdminClient adminClient) { Map<Properties, Integer> versionsNodeMap = new HashMap<Properties, Integer>(); for (Node node : adminClient.getAdminClientCluster().getNodes()) { String valueObject = getMetadataVersionsForNode(adminClient, node.getId()); Properties props = new Properties(); try { props.load(new ByteArrayInputStream(valueObject.getBytes())); } catch (IOException e) { System.err.println( "Error while parsing Metadata versions for node : " + node.getId() + ". Exception = \n"); e.printStackTrace(); System.exit(-1); } versionsNodeMap.put(props, node.getId()); } if (versionsNodeMap.keySet().size() > 1) { System.err.println("Mismatching versions detected !!!"); for (Entry<Properties, Integer> entry : versionsNodeMap.entrySet()) { System.out.println( "**************************** Node: " + entry.getValue() + " ****************************"); System.out.println(entry.getKey()); } } else { System.err.println("All the nodes have the same metadata versions ."); } } private static void synchronizeMetadataVersion(AdminClient adminClient, int baseNodeId) { String valueObject = getMetadataVersionsForNode(adminClient, baseNodeId); Properties props = new Properties(); try { props.load(new ByteArrayInputStream(valueObject.getBytes())); if (props.size() == 0) { System.err.println("The specified node does not have any versions metadata ! Exiting ..."); System.exit(-1); } adminClient.metadataMgmtOps.setMetadataversion(props); System.out.println("Metadata versions synchronized successfully."); } catch (IOException e) { System.err.println( "Error while retrieving Metadata versions from node : " + baseNodeId + ". Exception = \n"); e.printStackTrace(); System.exit(-1); } } private static void executeRollback(Integer nodeId, String storeName, long pushVersion, AdminClient adminClient) { if (nodeId < 0) { for (Node node : adminClient.getAdminClientCluster().getNodes()) { adminClient.readonlyOps.rollbackStore(node.getId(), storeName, pushVersion); } } else { adminClient.readonlyOps.rollbackStore(nodeId, storeName, pushVersion); } } private static void executeRepairJob(Integer nodeId, AdminClient adminClient) { if (nodeId < 0) { for (Node node : adminClient.getAdminClientCluster().getNodes()) { adminClient.storeMntOps.repairJob(node.getId()); } } else { adminClient.storeMntOps.repairJob(nodeId); } } private static void executePruneJob(Integer nodeId, AdminClient adminClient, List<String> stores) { if (nodeId < 0) { for (Node node : adminClient.getAdminClientCluster().getNodes()) { adminClient.storeMntOps.pruneJob(node.getId(), stores); } } else { adminClient.storeMntOps.pruneJob(nodeId, stores); } } private static void executePurgeSlops(AdminClient adminClient, List<Integer> nodesToPurge, Integer zoneToPurge, List<String> storesToPurge) { adminClient.storeMntOps.slopPurgeJob(nodesToPurge, zoneToPurge, storesToPurge); } private static void executeSetQuota(AdminClient adminClient, String storeName, String quotaType, String quotaValue) { if (!adminClient.helperOps.checkStoreExistsInCluster(storeName)) { Utils.croak("Store " + storeName + " not in cluster."); } adminClient.quotaMgmtOps.setQuota(storeName, quotaType, quotaValue); } private static void executeUnsetQuota(AdminClient adminClient, String storeName, String quotaType) { if (!adminClient.helperOps.checkStoreExistsInCluster(storeName)) { Utils.croak("Store " + storeName + " not in cluster."); } adminClient.quotaMgmtOps.unsetQuota(storeName, quotaType); } private static void executeGetQuota(AdminClient adminClient, String storeName, String quotaType) { if (!adminClient.helperOps.checkStoreExistsInCluster(storeName)) { Utils.croak("Store " + storeName + " not in cluster."); } Versioned<String> quotaVal = adminClient.quotaMgmtOps.getQuota(storeName, quotaType); if (quotaVal == null) { System.out.println("No quota set for " + quotaType + " on store " + storeName); } else { System.out.println( "Quota value for " + quotaType + " on store " + storeName + " : " + quotaVal.getValue()); } } public static void printHelp(PrintStream stream, OptionParser parser) throws IOException { stream.println("Commands supported"); stream.println("------------------"); stream.println("CHANGE METADATA"); stream.println("\t1) Get all metadata from all nodes"); stream.println("\t\t./bin/voldemort-admin-tool.sh --get-metadata --url [url]"); stream.println("\t2) Get metadata from all nodes"); stream.println( "\t\t./bin/voldemort-admin-tool.sh --get-metadata " + MetadataStore.METADATA_KEYS + " --url [url]"); stream.println("\t3) Get metadata from a particular node"); stream.println("\t\t./bin/voldemort-admin-tool.sh --get-metadata " + MetadataStore.METADATA_KEYS + " --url [url] --node [node-id]"); stream.println("\t4) Get metadata from a particular node and store to a directory"); stream.println("\t\t./bin/voldemort-admin-tool.sh --get-metadata " + MetadataStore.METADATA_KEYS + " --url [url] --node [node-id] --outdir [directory]"); stream.println("\t5) Set metadata on all nodes"); stream.println("\t\t./bin/voldemort-admin-tool.sh --set-metadata [" + MetadataStore.CLUSTER_KEY + ", " + MetadataStore.SERVER_STATE_KEY + ", " + MetadataStore.STORES_KEY + ", " + MetadataStore.REBALANCING_SOURCE_CLUSTER_XML + ", " + MetadataStore.REBALANCING_STEAL_INFO + "] --set-metadata-value [metadata-value] --url [url]"); stream.println("\t6) Set metadata for a particular node"); stream.println("\t\t./bin/voldemort-admin-tool.sh --set-metadata [" + MetadataStore.CLUSTER_KEY + ", " + MetadataStore.SERVER_STATE_KEY + ", " + MetadataStore.STORES_KEY + ", " + MetadataStore.REBALANCING_SOURCE_CLUSTER_XML + ", " + MetadataStore.REBALANCING_STEAL_INFO + "] --set-metadata-value [metadata-value] --url [url] --node [node-id]"); stream.println("\t7) Check if metadata is same on all nodes"); stream.println("\t\t./bin/voldemort-admin-tool.sh --check-metadata [" + MetadataStore.CLUSTER_KEY + ", " + MetadataStore.SERVER_STATE_KEY + ", " + MetadataStore.STORES_KEY + "] --url [url]"); stream.println("\t8) Clear rebalancing metadata [" + MetadataStore.SERVER_STATE_KEY + ", " + ", " + MetadataStore.REBALANCING_SOURCE_CLUSTER_XML + ", " + MetadataStore.REBALANCING_STEAL_INFO + "] on all node "); stream.println("\t\t./bin/voldemort-admin-tool.sh --clear-rebalancing-metadata --url [url]"); stream.println("\t9) Clear rebalancing metadata [" + MetadataStore.SERVER_STATE_KEY + ", " + ", " + MetadataStore.REBALANCING_SOURCE_CLUSTER_XML + ", " + MetadataStore.REBALANCING_STEAL_INFO + "] on a particular node "); stream.println( "\t\t./bin/voldemort-admin-tool.sh --clear-rebalancing-metadata --url [url] --node [node-id]"); stream.println("\t10) View detailed routing information for a given set of keys."); stream.println( "bin/voldemort-admin-tool.sh --url <url> --show-routing-plan key1,key2,.. --store <store-name>"); stream.println(); stream.println("ADD / DELETE STORES"); stream.println("\t1) Add store(s) on all nodes"); stream.println( "\t\t./bin/voldemort-admin-tool.sh --add-stores [xml file with store(s) to add] --url [url]"); stream.println("\t2) Add store(s) on a single node"); stream.println( "\t\t./bin/voldemort-admin-tool.sh --add-stores [xml file with store(s) to add] --url [url] --node [node-id]"); stream.println("\t3) Delete store on all nodes"); stream.println("\t\t./bin/voldemort-admin-tool.sh --delete-store [store-name] --url [url]"); stream.println("\t4) Delete store on a single node"); stream.println( "\t\t./bin/voldemort-admin-tool.sh --delete-store [store-name] --url [url] --node [node-id]"); stream.println("\t5) Delete the contents of the store on all nodes"); stream.println("\t\t./bin/voldemort-admin-tool.sh --truncate [store-name] --url [url]"); stream.println("\t6) Delete the contents of the store on a single node"); stream.println("\t\t./bin/voldemort-admin-tool.sh --truncate [store-name] --url [url] --node [node-id]"); stream.println("\t7) Delete the contents of some partitions on a single node"); stream.println( "\t\t./bin/voldemort-admin-tool.sh --delete-partitions [comma-separated list of partitions] --url [url] --node [node-id]"); stream.println("\t8) Delete the contents of some partitions ( of some stores ) on a single node"); stream.println( "\t\t./bin/voldemort-admin-tool.sh --delete-partitions [comma-separated list of partitions] --url [url] --node [node-id] --stores [comma-separated list of store names]"); stream.println(); stream.println("STREAM DATA"); stream.println("\t1) Fetch keys from a set of partitions [ all stores ] on a node ( binary dump )"); stream.println( "\t\t./bin/voldemort-admin-tool.sh --fetch-keys [comma-separated list of partitions with no space] --url [url] --node [node-id]"); stream.println("\t2) Fetch keys from a set of partitions [ all stores ] on a node ( ascii enabled )"); stream.println( "\t\t./bin/voldemort-admin-tool.sh --fetch-keys [comma-separated list of partitions with no space] --url [url] --node [node-id] --ascii"); stream.println("\t3) Fetch entries from a set of partitions [ all stores ] on a node ( binary dump )"); stream.println( "\t\t./bin/voldemort-admin-tool.sh --fetch-entries [comma-separated list of partitions with no space] --url [url] --node [node-id]"); stream.println("\t4) Fetch entries from a set of partitions [ all stores ] on a node ( ascii enabled )"); stream.println( "\t\t./bin/voldemort-admin-tool.sh --fetch-entries [comma-separated list of partitions with no space] --url [url] --node [node-id] --ascii"); stream.println( "\t5) Fetch entries from a set of partitions [ all stores ] on a node ( ascii enabled ) and output to a folder"); stream.println( "\t\t./bin/voldemort-admin-tool.sh --fetch-entries [comma-separated list of partitions with no space] --url [url] --node [node-id] --ascii --outdir [directory]"); stream.println("\t6) Fetch entries from a set of partitions and some stores on a node ( ascii enabled )"); stream.println( "\t\t./bin/voldemort-admin-tool.sh --fetch-entries [comma-separated list of partitions with no space] --url [url] --node [node-id] --ascii --stores [comma-separated list of store names] "); stream.println("\t7) Fetch all keys on a particular node"); stream.println("\t\t./bin/voldemort-admin-tool.sh --fetch-keys --url [url] --node [node-id]"); stream.println("\t8) Fetch all entries on a particular node"); stream.println("\t\t./bin/voldemort-admin-tool.sh --fetch-entries --url [url] --node [node-id]"); stream.println("\t9) Update entries for a set of stores using the output from a binary dump fetch entries"); stream.println( "\t\t./bin/voldemort-admin-tool.sh --update-entries [folder path from output of --fetch-entries --outdir] --url [url] --node [node-id] --stores [comma-separated list of store names]"); stream.println("\t10.a) Query stores for a set of keys on a specific node, key is in hex format"); stream.println( "\t\t./bin/voldemort-admin-tool.sh --query-key [hex_key] --query-key-format hex --url [url] --node [node-id] --stores [comma-separated list of store names]"); stream.println("\t11.a) Query stores for a set of keys on all nodes, key is in hex format"); stream.println( "\t\t./bin/voldemort-admin-tool.sh --query-key [hex_key] --query-key-format hex --url [url] --stores [comma-separated list of store names]"); stream.println( "\t12.a) Query stores for a set of keys on a specific node, in readable format. JSON string must be between quotation marks with inside quotation marks escaped"); stream.println( "\t\t./bin/voldemort-admin-tool.sh --query-key [readable_key] --query-key-format readable --url [url] --node [node-id] --stores [comma-separated list of store names]"); stream.println( "\t13) Mirror data from another voldemort server (possibly in another cluster) for specified stores"); stream.println( "\t\t./bin/voldemort-admin-tool.sh --mirror-from-url [bootstrap url to mirror from] --mirror-node [node to mirror from] --url [url] --node [node-id] --stores [comma-separated-list-of-store-names]"); stream.println( "\t14) Mirror data from another voldemort server (possibly in another cluster) for all stores in current cluster"); stream.println( "\t\t./bin/voldemort-admin-tool.sh --mirror-from-url [bootstrap url to mirror from] --mirror-node [node to mirror from] --url [url] --node [node-id]"); stream.println("\t15) Fetch all orphaned keys on a particular node"); stream.println( "\t\t./bin/voldemort-admin-tool.sh --fetch-keys --url [url] --node [node-id] --fetch-orphaned"); stream.println("\t16) Fetch all orphaned entries on a particular node"); stream.println( "\t\t./bin/voldemort-admin-tool.sh --fetch-entries --url [url] --node [node-id] --fetch-orphaned"); stream.println(); stream.println("READ-ONLY OPERATIONS"); stream.println("\t1) Retrieve metadata information of read-only data for a particular node and all stores"); stream.println( "\t\t./bin/voldemort-admin-tool.sh --ro-metadata [current | max | storage-format] --url [url] --node [node-id]"); stream.println("\t2) Retrieve metadata information of read-only data for all nodes and a set of store"); stream.println( "\t\t./bin/voldemort-admin-tool.sh --ro-metadata [current | max | storage-format] --url [url] --stores [comma-separated list of store names]"); stream.println(); stream.println("ASYNC JOBS"); stream.println("\t1) Get a list of async jobs on all nodes"); stream.println("\t\t./bin/voldemort-admin-tool.sh --async get --url [url]"); stream.println("\t2) Get a list of async jobs on a particular node"); stream.println("\t\t./bin/voldemort-admin-tool.sh --async get --url [url] --node [node-id]"); stream.println("\t3) Stop a list of async jobs on a particular node"); stream.println( "\t\t./bin/voldemort-admin-tool.sh --async stop --async-id [comma-separated list of async job id] --url [url] --node [node-id]"); stream.println(); stream.println("OTHERS"); stream.println("\t1) Restore a particular node completely from its replicas"); stream.println("\t\t./bin/voldemort-admin-tool.sh --restore --url [url] --node [node-id]"); stream.println( "\t2) Restore a particular node completely from its replicas ( with increased parallelism - 10 ) "); stream.println("\t\t./bin/voldemort-admin-tool.sh --restore 10 --url [url] --node [node-id]"); stream.println("\t3) Clean a node after rebalancing is done"); stream.println("\t\t./bin/voldemort-admin-tool.sh --repair-job --url [url] --node [node-id]"); stream.println("\t4) Backup bdb data natively"); stream.println("\t\t./bin/voldemort-admin-tool.sh --native-backup [store] --backup-dir [outdir] " + "--backup-timeout [mins] [--backup-verify] [--backup-incremental] --url [url] --node [node-id]"); stream.println("\t5) Rollback a read-only store to the specified push version"); stream.println( "\t\t./bin/voldemort-admin-tool.sh --rollback [store-name] --url [url] --node [node-id] --version [version-num] "); stream.println("\t7) Prune data resulting from versioned puts, during rebalancing"); stream.println( "\t\t./bin/voldemort-admin-tool.sh --prune-job --url [url] --node [node-id] --stores [stores_list]"); stream.println("\t8) Purge slops based on criteria"); stream.println( "\t\t./bin/voldemort-admin-tool.sh --purge-slops --url [url] --nodes [destination-nodes-list] --stores [stores_list] --zone [destination-zone]"); stream.println("\t9) Set Quota limits on the servers. One of " + QuotaUtils.validQuotaTypes()); stream.println( "\t\t bin/voldemort-admin-tool.sh --url [url] --set-quota [quota-type] --quota-value [value] --store [store-name]"); stream.println("\t10) Unset Quota limits on the servers. One of " + QuotaUtils.validQuotaTypes()); stream.println( "\t\t bin/voldemort-admin-tool.sh --url [url] --unset-quota [quota-type] --store [store-name]"); stream.println("\t11) Get Quota limits on the servers. One of " + QuotaUtils.validQuotaTypes()); stream.println( "\t\t bin/voldemort-admin-tool.sh --url [url] --get-quota [quota-type] --store [store-name]"); parser.printHelpOn(stream); } private static void executeAsync(Integer nodeId, AdminClient adminClient, String asyncKey, List<Integer> asyncIdsToStop) { if (asyncKey.compareTo("get") == 0) { List<Integer> nodeIds = Lists.newArrayList(); if (nodeId < 0) { for (Node node : adminClient.getAdminClientCluster().getNodes()) { nodeIds.add(node.getId()); } } else { nodeIds.add(nodeId); } // Print the job information for (int currentNodeId : nodeIds) { System.out.println("Retrieving async jobs from node " + currentNodeId); List<Integer> asyncIds = adminClient.rpcOps.getAsyncRequestList(currentNodeId); System.out.println("Async Job Ids on node " + currentNodeId + " : " + asyncIds); for (int asyncId : asyncIds) { System.out.println("Async Job Id " + asyncId + " ] " + adminClient.rpcOps.getAsyncRequestStatus(currentNodeId, asyncId)); System.out.println(); } } } else if (asyncKey.compareTo("stop") == 0) { if (nodeId < 0) { throw new VoldemortException("Cannot stop job ids without node id"); } if (asyncIdsToStop == null || asyncIdsToStop.size() == 0) { throw new VoldemortException("Async ids cannot be null / zero"); } for (int asyncId : asyncIdsToStop) { System.out.println("Stopping async id " + asyncId); adminClient.rpcOps.stopAsyncRequest(nodeId, asyncId); System.out.println("Stopped async id " + asyncId); } } else { throw new VoldemortException("Unsupported async operation type " + asyncKey); } } private static void executeClearRebalancing(int nodeId, AdminClient adminClient) { System.out.println( "Setting " + MetadataStore.SERVER_STATE_KEY + " to " + MetadataStore.VoldemortState.NORMAL_SERVER); executeSetMetadata(nodeId, adminClient, MetadataStore.SERVER_STATE_KEY, MetadataStore.VoldemortState.NORMAL_SERVER.toString()); RebalancerState state = RebalancerState.create("[]"); System.out.println("Cleaning up " + MetadataStore.REBALANCING_STEAL_INFO + " to " + state.toJsonString()); executeSetMetadata(nodeId, adminClient, MetadataStore.REBALANCING_STEAL_INFO, state.toJsonString()); System.out.println("Cleaning up " + MetadataStore.REBALANCING_SOURCE_CLUSTER_XML + " to empty string"); executeSetMetadata(nodeId, adminClient, MetadataStore.REBALANCING_SOURCE_CLUSTER_XML, ""); } private static void executeCheckMetadata(AdminClient adminClient, String metadataKey) { Set<Object> metadataValues = Sets.newHashSet(); for (Node node : adminClient.getAdminClientCluster().getNodes()) { System.out.println(node.getHost() + ":" + node.getId()); Versioned<String> versioned = adminClient.metadataMgmtOps.getRemoteMetadata(node.getId(), metadataKey); if (versioned == null || versioned.getValue() == null) { throw new VoldemortException("Value returned from node " + node.getId() + " was null"); } else { if (metadataKey.compareTo(MetadataStore.CLUSTER_KEY) == 0 || metadataKey.compareTo(MetadataStore.REBALANCING_SOURCE_CLUSTER_XML) == 0) { metadataValues.add(new ClusterMapper().readCluster(new StringReader(versioned.getValue()))); } else if (metadataKey.compareTo(MetadataStore.STORES_KEY) == 0) { metadataValues.add( new StoreDefinitionsMapper().readStoreList(new StringReader(versioned.getValue()))); } else if (metadataKey.compareTo(MetadataStore.SERVER_STATE_KEY) == 0) { metadataValues.add(VoldemortState.valueOf(versioned.getValue())); } else { throw new VoldemortException("Incorrect metadata key"); } } } if (metadataValues.size() == 1) { System.out.println("true"); } else { System.out.println("false"); } } /* * Update <cluster.xml,stores.xml> pair atomically */ public static void executeSetMetadataPair(Integer nodeId, AdminClient adminClient, String clusterKey, Object clusterValue, String storesKey, Object storesValue) { List<Integer> nodeIds = Lists.newArrayList(); VectorClock updatedClusterVersion = null; VectorClock updatedStoresVersion = null; if (nodeId < 0) { for (Node node : adminClient.getAdminClientCluster().getNodes()) { nodeIds.add(node.getId()); if (updatedClusterVersion == null && updatedStoresVersion == null) { updatedClusterVersion = (VectorClock) adminClient.metadataMgmtOps .getRemoteMetadata(node.getId(), clusterKey).getVersion(); updatedStoresVersion = (VectorClock) adminClient.metadataMgmtOps .getRemoteMetadata(node.getId(), storesKey).getVersion(); } else { updatedClusterVersion = updatedClusterVersion.merge((VectorClock) adminClient.metadataMgmtOps .getRemoteMetadata(node.getId(), clusterKey).getVersion()); updatedStoresVersion = updatedStoresVersion.merge((VectorClock) adminClient.metadataMgmtOps .getRemoteMetadata(node.getId(), storesKey).getVersion()); } } // TODO: This will work for now but we should take a step back and // think about a uniform clock for the metadata values. updatedClusterVersion = updatedClusterVersion.incremented(0, System.currentTimeMillis()); updatedStoresVersion = updatedStoresVersion.incremented(0, System.currentTimeMillis()); } else { updatedClusterVersion = ((VectorClock) adminClient.metadataMgmtOps.getRemoteMetadata(nodeId, clusterKey) .getVersion()).incremented(nodeId, System.currentTimeMillis()); updatedStoresVersion = ((VectorClock) adminClient.metadataMgmtOps.getRemoteMetadata(nodeId, storesKey) .getVersion()).incremented(nodeId, System.currentTimeMillis()); nodeIds.add(nodeId); } adminClient.metadataMgmtOps.updateRemoteMetadataPair(nodeIds, clusterKey, Versioned.value(clusterValue.toString(), updatedClusterVersion), storesKey, Versioned.value(storesValue.toString(), updatedStoresVersion)); } public static void executeSetMetadata(Integer nodeId, AdminClient adminClient, String key, Object value) { List<Integer> nodeIds = Lists.newArrayList(); VectorClock updatedVersion = null; if (nodeId < 0) { for (Node node : adminClient.getAdminClientCluster().getNodes()) { nodeIds.add(node.getId()); if (updatedVersion == null) { updatedVersion = (VectorClock) adminClient.metadataMgmtOps.getRemoteMetadata(node.getId(), key) .getVersion(); } else { updatedVersion = updatedVersion.merge((VectorClock) adminClient.metadataMgmtOps .getRemoteMetadata(node.getId(), key).getVersion()); } } // Bump up version on node 0 updatedVersion = updatedVersion.incremented(0, System.currentTimeMillis()); } else { Versioned<String> currentValue = adminClient.metadataMgmtOps.getRemoteMetadata(nodeId, key); updatedVersion = ((VectorClock) currentValue.getVersion()).incremented(nodeId, System.currentTimeMillis()); nodeIds.add(nodeId); } adminClient.metadataMgmtOps.updateRemoteMetadata(nodeIds, key, Versioned.value(value.toString(), updatedVersion)); } private static void executeUpdateMetadataVersionsOnStores(AdminClient adminClient, List<StoreDefinition> oldStoreDefs, List<StoreDefinition> newStoreDefs) { Set<String> storeNamesUnion = new HashSet<String>(); Map<String, StoreDefinition> oldStoreDefinitionMap = new HashMap<String, StoreDefinition>(); Map<String, StoreDefinition> newStoreDefinitionMap = new HashMap<String, StoreDefinition>(); List<String> storesChanged = new ArrayList<String>(); for (StoreDefinition storeDef : oldStoreDefs) { String storeName = storeDef.getName(); storeNamesUnion.add(storeName); oldStoreDefinitionMap.put(storeName, storeDef); } for (StoreDefinition storeDef : newStoreDefs) { String storeName = storeDef.getName(); storeNamesUnion.add(storeName); newStoreDefinitionMap.put(storeName, storeDef); } for (String storeName : storeNamesUnion) { StoreDefinition oldStoreDef = oldStoreDefinitionMap.get(storeName); StoreDefinition newStoreDef = newStoreDefinitionMap.get(storeName); if (oldStoreDef == null && newStoreDef != null || oldStoreDef != null && newStoreDef == null || oldStoreDef != null && newStoreDef != null && !oldStoreDef.equals(newStoreDef)) { storesChanged.add(storeName); } } System.out.println("Updating metadata version for the following stores: " + storesChanged); try { adminClient.metadataMgmtOps.updateMetadataversion(storesChanged); } catch (Exception e) { System.err.println("Error while updating metadata version for the specified store."); } } private static void executeROMetadata(Integer nodeId, AdminClient adminClient, List<String> storeNames, String type) { Map<String, Long> storeToValue = Maps.newHashMap(); if (storeNames == null) { // Retrieve list of read-only stores storeNames = Lists.newArrayList(); for (StoreDefinition storeDef : adminClient.metadataMgmtOps .getRemoteStoreDefList(nodeId > 0 ? nodeId : 0).getValue()) { if (storeDef.getType().compareTo(ReadOnlyStorageConfiguration.TYPE_NAME) == 0) { storeNames.add(storeDef.getName()); } } } List<Integer> nodeIds = Lists.newArrayList(); if (nodeId < 0) { for (Node node : adminClient.getAdminClientCluster().getNodes()) { nodeIds.add(node.getId()); } } else { nodeIds.add(nodeId); } for (int currentNodeId : nodeIds) { System.out.println(adminClient.getAdminClientCluster().getNodeById(currentNodeId).getHost() + ":" + adminClient.getAdminClientCluster().getNodeById(currentNodeId).getId()); if (type.compareTo("max") == 0) { storeToValue = adminClient.readonlyOps.getROMaxVersion(currentNodeId, storeNames); } else if (type.compareTo("current") == 0) { storeToValue = adminClient.readonlyOps.getROCurrentVersion(currentNodeId, storeNames); } else if (type.compareTo("storage-format") == 0) { Map<String, String> storeToStorageFormat = adminClient.readonlyOps.getROStorageFormat(currentNodeId, storeNames); for (String storeName : storeToStorageFormat.keySet()) { System.out.println(storeName + ":" + storeToStorageFormat.get(storeName)); } continue; } else { System.err.println("Unsupported operation, only max, current or storage-format allowed"); return; } for (String storeName : storeToValue.keySet()) { System.out.println(storeName + ":" + storeToValue.get(storeName)); } } } public static void executeGetMetadata(Integer nodeId, AdminClient adminClient, String metadataKey, String outputDir) throws IOException { File directory = null; if (outputDir != null) { directory = new File(outputDir); if (!(directory.exists() || directory.mkdir())) { Utils.croak("Can't find or create directory " + outputDir); } } List<Integer> nodeIds = Lists.newArrayList(); if (nodeId < 0) { for (Node node : adminClient.getAdminClientCluster().getNodes()) { nodeIds.add(node.getId()); } } else { nodeIds.add(nodeId); } List<String> metadataKeys = Lists.newArrayList(); if (metadataKey.compareTo(ALL_METADATA) == 0) { for (Object key : MetadataStore.METADATA_KEYS) { metadataKeys.add((String) key); } } else { metadataKeys.add(metadataKey); } for (Integer currentNodeId : nodeIds) { System.out.println(adminClient.getAdminClientCluster().getNodeById(currentNodeId).getHost() + ":" + adminClient.getAdminClientCluster().getNodeById(currentNodeId).getId()); for (String key : metadataKeys) { System.out.println("Key - " + key); Versioned<String> versioned = null; try { versioned = adminClient.metadataMgmtOps.getRemoteMetadata(currentNodeId, key); } catch (Exception e) { System.out.println("Error in retrieving " + e.getMessage()); System.out.println(); continue; } if (versioned == null) { if (directory == null) { System.out.println("null"); System.out.println(); } else { FileUtils.writeStringToFile(new File(directory, key + "_" + currentNodeId), ""); } } else { if (directory == null) { System.out.println(versioned.getVersion()); System.out.print(": "); System.out.println(versioned.getValue()); System.out.println(); } else { FileUtils.writeStringToFile(new File(directory, key + "_" + currentNodeId), versioned.getValue()); } } } } } private static void executeDeleteStore(AdminClient adminClient, String storeName, int nodeId) { System.out.println("Deleting " + storeName); if (nodeId == -1) { adminClient.storeMgmtOps.deleteStore(storeName); } else { adminClient.storeMgmtOps.deleteStore(storeName, nodeId); } } private static void executeTruncateStore(int nodeId, AdminClient adminClient, String storeName) { List<Integer> nodeIds = Lists.newArrayList(); if (nodeId < 0) { for (Node node : adminClient.getAdminClientCluster().getNodes()) { nodeIds.add(node.getId()); } } else { nodeIds.add(nodeId); } for (Integer currentNodeId : nodeIds) { System.out.println("Truncating " + storeName + " on node " + currentNodeId); adminClient.storeMntOps.truncate(currentNodeId, storeName); } } private static void executeAddStores(AdminClient adminClient, String storesXml, int nodeId) throws IOException { List<StoreDefinition> storeDefinitionList = new StoreDefinitionsMapper().readStoreList(new File(storesXml)); for (StoreDefinition storeDef : storeDefinitionList) { System.out.println("Adding " + storeDef.getName()); if (-1 != nodeId) adminClient.storeMgmtOps.addStore(storeDef, nodeId); else adminClient.storeMgmtOps.addStore(storeDef); } } private static void executeFetchEntries(Integer nodeId, AdminClient adminClient, List<Integer> partitionIdList, String outputDir, List<String> storeNames, boolean useAscii, boolean fetchOrphaned) throws IOException { List<StoreDefinition> storeDefinitionList = adminClient.metadataMgmtOps.getRemoteStoreDefList(nodeId) .getValue(); HashMap<String, StoreDefinition> storeDefinitionMap = Maps.newHashMap(); for (StoreDefinition storeDefinition : storeDefinitionList) { storeDefinitionMap.put(storeDefinition.getName(), storeDefinition); } File directory = null; if (outputDir != null) { directory = new File(outputDir); if (!(directory.exists() || directory.mkdir())) { Utils.croak("Can't find or create directory " + outputDir); } } List<String> stores = storeNames; if (stores == null) { // when no stores specified, all user defined store will be fetched, // but not system stores. stores = Lists.newArrayList(); stores.addAll(storeDefinitionMap.keySet()); } else { // add system stores to the map so they can be fetched when // specified explicitly storeDefinitionMap.putAll(getSystemStoreDefs()); } // Pick up all the partitions if (partitionIdList == null) { partitionIdList = Lists.newArrayList(); for (Node node : adminClient.getAdminClientCluster().getNodes()) { partitionIdList.addAll(node.getPartitionIds()); } } StoreDefinition storeDefinition = null; for (String store : stores) { storeDefinition = storeDefinitionMap.get(store); if (null == storeDefinition) { System.out.println("No store found under the name \'" + store + "\'"); continue; } Iterator<Pair<ByteArray, Versioned<byte[]>>> entriesIteratorRef = null; if (fetchOrphaned) { System.out.println("Fetching orphaned entries of " + store); entriesIteratorRef = adminClient.bulkFetchOps.fetchOrphanedEntries(nodeId, store); } else { System.out.println( "Fetching entries in partitions " + Joiner.on(", ").join(partitionIdList) + " of " + store); entriesIteratorRef = adminClient.bulkFetchOps.fetchEntries(nodeId, store, partitionIdList, null, false); } final Iterator<Pair<ByteArray, Versioned<byte[]>>> entriesIterator = entriesIteratorRef; File outputFile = null; if (directory != null) { outputFile = new File(directory, store + ".entries"); } if (useAscii) { // k-v serializer SerializerDefinition keySerializerDef = storeDefinition.getKeySerializer(); SerializerDefinition valueSerializerDef = storeDefinition.getValueSerializer(); SerializerFactory serializerFactory = new DefaultSerializerFactory(); @SuppressWarnings("unchecked") final Serializer<Object> keySerializer = (Serializer<Object>) serializerFactory .getSerializer(keySerializerDef); @SuppressWarnings("unchecked") final Serializer<Object> valueSerializer = (Serializer<Object>) serializerFactory .getSerializer(valueSerializerDef); // compression strategy final CompressionStrategy keyCompressionStrategy; final CompressionStrategy valueCompressionStrategy; if (keySerializerDef != null && keySerializerDef.hasCompression()) { keyCompressionStrategy = new CompressionStrategyFactory() .get(keySerializerDef.getCompression()); } else { keyCompressionStrategy = null; } if (valueSerializerDef != null && valueSerializerDef.hasCompression()) { valueCompressionStrategy = new CompressionStrategyFactory() .get(valueSerializerDef.getCompression()); } else { valueCompressionStrategy = null; } writeAscii(outputFile, new Writable() { @Override public void writeTo(BufferedWriter out) throws IOException { while (entriesIterator.hasNext()) { final JsonGenerator generator = new JsonFactory(new ObjectMapper()) .createJsonGenerator(out); Pair<ByteArray, Versioned<byte[]>> kvPair = entriesIterator.next(); byte[] keyBytes = kvPair.getFirst().get(); byte[] valueBytes = kvPair.getSecond().getValue(); VectorClock version = (VectorClock) kvPair.getSecond().getVersion(); Object keyObject = keySerializer.toObject((null == keyCompressionStrategy) ? keyBytes : keyCompressionStrategy.inflate(keyBytes)); Object valueObject = valueSerializer .toObject((null == valueCompressionStrategy) ? valueBytes : valueCompressionStrategy.inflate(valueBytes)); if (keyObject instanceof GenericRecord) { out.write(keyObject.toString()); } else { generator.writeObject(keyObject); } out.write(' ' + version.toString() + ' '); if (valueObject instanceof GenericRecord) { out.write(valueObject.toString()); } else { generator.writeObject(valueObject); } out.write('\n'); } } }); } else { writeBinary(outputFile, new Printable() { @Override public void printTo(DataOutputStream out) throws IOException { while (entriesIterator.hasNext()) { Pair<ByteArray, Versioned<byte[]>> kvPair = entriesIterator.next(); byte[] keyBytes = kvPair.getFirst().get(); VectorClock clock = ((VectorClock) kvPair.getSecond().getVersion()); byte[] valueBytes = kvPair.getSecond().getValue(); out.writeChars(ByteUtils.toHexString(keyBytes)); out.writeChars(","); out.writeChars(clock.toString()); out.writeChars(","); out.writeChars(ByteUtils.toHexString(valueBytes)); out.writeChars("\n"); } } }); } if (outputFile != null) System.out.println("Fetched keys from " + store + " to " + outputFile); } } private static Map<String, StoreDefinition> getSystemStoreDefs() { Map<String, StoreDefinition> sysStoreDefMap = Maps.newHashMap(); List<StoreDefinition> storesDefs = SystemStoreConstants.getAllSystemStoreDefs(); for (StoreDefinition def : storesDefs) { sysStoreDefMap.put(def.getName(), def); } return sysStoreDefMap; } private static void executeUpdateEntries(Integer nodeId, AdminClient adminClient, List<String> storeNames, String inputDirPath) throws IOException { List<StoreDefinition> storeDefinitionList = adminClient.metadataMgmtOps.getRemoteStoreDefList(nodeId) .getValue(); Map<String, StoreDefinition> storeDefinitionMap = Maps.newHashMap(); for (StoreDefinition storeDefinition : storeDefinitionList) { storeDefinitionMap.put(storeDefinition.getName(), storeDefinition); } File inputDir = new File(inputDirPath); if (!inputDir.exists()) { throw new FileNotFoundException("input directory " + inputDirPath + " doesn't exist"); } if (storeNames == null) { storeNames = Lists.newArrayList(); for (File storeFile : inputDir.listFiles()) { String fileName = storeFile.getName(); if (fileName.endsWith(".entries")) { int extPosition = fileName.lastIndexOf(".entries"); storeNames.add(fileName.substring(0, extPosition)); } } } for (String storeName : storeNames) { Iterator<Pair<ByteArray, Versioned<byte[]>>> iterator = readEntriesBinary(inputDir, storeName); adminClient.streamingOps.updateEntries(nodeId, storeName, iterator, null); } } private static Iterator<Pair<ByteArray, Versioned<byte[]>>> readEntriesBinary(File inputDir, String storeName) throws IOException { File inputFile = new File(inputDir, storeName + ".entries"); if (!inputFile.exists()) { throw new FileNotFoundException("File " + inputFile.getAbsolutePath() + " does not exist!"); } final DataInputStream dis = new DataInputStream(new BufferedInputStream(new FileInputStream(inputFile))); return new AbstractIterator<Pair<ByteArray, Versioned<byte[]>>>() { @Override protected Pair<ByteArray, Versioned<byte[]>> computeNext() { try { int length = dis.readInt(); byte[] keyBytes = new byte[length]; ByteUtils.read(dis, keyBytes); length = dis.readInt(); byte[] versionBytes = new byte[length]; ByteUtils.read(dis, versionBytes); length = dis.readInt(); byte[] valueBytes = new byte[length]; ByteUtils.read(dis, valueBytes); ByteArray key = new ByteArray(keyBytes); VectorClock version = new VectorClock(versionBytes); Versioned<byte[]> value = new Versioned<byte[]>(valueBytes, version); return new Pair<ByteArray, Versioned<byte[]>>(key, value); } catch (EOFException e) { try { dis.close(); } catch (IOException ie) { ie.printStackTrace(); } return endOfData(); } catch (IOException e) { try { dis.close(); } catch (IOException ie) { ie.printStackTrace(); } throw new VoldemortException("Error reading from input file ", e); } } }; } private static void executeFetchKeys(Integer nodeId, AdminClient adminClient, List<Integer> partitionIdList, String outputDir, List<String> storeNames, boolean useAscii, boolean fetchOrphaned) throws IOException { List<StoreDefinition> storeDefinitionList = adminClient.metadataMgmtOps.getRemoteStoreDefList(nodeId) .getValue(); Map<String, StoreDefinition> storeDefinitionMap = Maps.newHashMap(); for (StoreDefinition storeDefinition : storeDefinitionList) { storeDefinitionMap.put(storeDefinition.getName(), storeDefinition); } File directory = null; if (outputDir != null) { directory = new File(outputDir); if (!(directory.exists() || directory.mkdir())) { Utils.croak("Can't find or create directory " + outputDir); } } List<String> stores = storeNames; if (stores == null) { stores = Lists.newArrayList(); stores.addAll(storeDefinitionMap.keySet()); } else { // add system stores to the map so they can be fetched when // specified explicitly storeDefinitionMap.putAll(getSystemStoreDefs()); } // Pick up all the partitions if (partitionIdList == null) { partitionIdList = Lists.newArrayList(); for (Node node : adminClient.getAdminClientCluster().getNodes()) { partitionIdList.addAll(node.getPartitionIds()); } } StoreDefinition storeDefinition = null; for (String store : stores) { storeDefinition = storeDefinitionMap.get(store); if (null == storeDefinition) { System.out.println("No store found under the name \'" + store + "\'"); continue; } Iterator<ByteArray> keyIteratorRef = null; if (fetchOrphaned) { System.out.println("Fetching orphaned keys of " + store); keyIteratorRef = adminClient.bulkFetchOps.fetchOrphanedKeys(nodeId, store); } else { System.out.println( "Fetching keys in partitions " + Joiner.on(", ").join(partitionIdList) + " of " + store); keyIteratorRef = adminClient.bulkFetchOps.fetchKeys(nodeId, store, partitionIdList, null, false); } File outputFile = null; if (directory != null) { outputFile = new File(directory, store + ".keys"); } final Iterator<ByteArray> keyIterator = keyIteratorRef; if (useAscii) { final SerializerDefinition serializerDef = storeDefinition.getKeySerializer(); final SerializerFactory serializerFactory = new DefaultSerializerFactory(); @SuppressWarnings("unchecked") final Serializer<Object> serializer = (Serializer<Object>) serializerFactory .getSerializer(serializerDef); final CompressionStrategy keysCompressionStrategy; if (serializerDef != null && serializerDef.hasCompression()) { keysCompressionStrategy = new CompressionStrategyFactory().get(serializerDef.getCompression()); } else { keysCompressionStrategy = null; } writeAscii(outputFile, new Writable() { @Override public void writeTo(BufferedWriter out) throws IOException { while (keyIterator.hasNext()) { final JsonGenerator generator = new JsonFactory(new ObjectMapper()) .createJsonGenerator(out); byte[] keyBytes = keyIterator.next().get(); Object keyObject = serializer.toObject((null == keysCompressionStrategy) ? keyBytes : keysCompressionStrategy.inflate(keyBytes)); if (keyObject instanceof GenericRecord) { out.write(keyObject.toString()); } else { generator.writeObject(keyObject); } out.write('\n'); } } }); } else { writeBinary(outputFile, new Printable() { @Override public void printTo(DataOutputStream out) throws IOException { while (keyIterator.hasNext()) { byte[] keyBytes = keyIterator.next().get(); out.writeChars(ByteUtils.toHexString(keyBytes) + "\n"); } } }); } if (outputFile != null) System.out.println("Fetched keys from " + store + " to " + outputFile); } } private abstract static class Printable { public abstract void printTo(DataOutputStream out) throws IOException; } private abstract static class Writable { public abstract void writeTo(BufferedWriter out) throws IOException; } private static void writeBinary(File outputFile, Printable printable) throws IOException { OutputStream outputStream = null; if (outputFile == null) { outputStream = new FilterOutputStream(System.out) { @Override public void close() throws IOException { flush(); } }; } else { outputStream = new FileOutputStream(outputFile); } DataOutputStream dataOutputStream = new DataOutputStream(new BufferedOutputStream(outputStream)); try { printable.printTo(dataOutputStream); } finally { dataOutputStream.close(); } } private static void writeAscii(File outputFile, Writable writable) throws IOException { Writer writer = null; if (outputFile == null) { writer = new OutputStreamWriter(new FilterOutputStream(System.out) { @Override public void close() throws IOException { flush(); } }); } else { writer = new FileWriter(outputFile); } BufferedWriter bufferedWriter = new BufferedWriter(writer); try { writable.writeTo(bufferedWriter); } finally { bufferedWriter.close(); } } private static void executeDeletePartitions(Integer nodeId, AdminClient adminClient, List<Integer> partitionIdList, List<String> storeNames) { List<String> stores = storeNames; if (stores == null) { stores = Lists.newArrayList(); List<StoreDefinition> storeDefinitionList = adminClient.metadataMgmtOps.getRemoteStoreDefList(nodeId) .getValue(); for (StoreDefinition storeDefinition : storeDefinitionList) { stores.add(storeDefinition.getName()); } } for (String store : stores) { System.out.println("Deleting partitions " + Joiner.on(", ").join(partitionIdList) + " of " + store); adminClient.storeMntOps.deletePartitions(nodeId, store, partitionIdList, null); } } private static void executeQueryKey(final Integer nodeId, AdminClient adminClient, List<String> storeNames, String keyString, String keyFormat) throws IOException { // decide queryNode for storeDef int storeDefNodeId; if (nodeId < 0) { Iterator<Node> nodeIterator = adminClient.getAdminClientCluster().getNodes().iterator(); if (!nodeIterator.hasNext()) { throw new VoldemortException("No nodes in this cluster"); } storeDefNodeId = nodeIterator.next().getId(); } else { storeDefNodeId = nodeId; } // decide queryingNode(s) for Key List<Integer> queryingNodes = new ArrayList<Integer>(); if (nodeId < 0) { // means all nodes for (Node node : adminClient.getAdminClientCluster().getNodes()) { queryingNodes.add(node.getId()); } } else { queryingNodes.add(nodeId); } // get basic info List<StoreDefinition> storeDefinitionList = adminClient.metadataMgmtOps .getRemoteStoreDefList(storeDefNodeId).getValue(); Map<String, StoreDefinition> storeDefinitions = new HashMap<String, StoreDefinition>(); for (StoreDefinition storeDef : storeDefinitionList) { storeDefinitions.put(storeDef.getName(), storeDef); } BufferedWriter out = new BufferedWriter(new OutputStreamWriter(System.out)); // iterate through stores for (final String storeName : storeNames) { // store definition StoreDefinition storeDefinition = storeDefinitions.get(storeName); if (storeDefinition == null) { throw new StoreNotFoundException("Store " + storeName + " not found"); } out.write("STORE_NAME: " + storeDefinition.getName() + "\n"); // k-v serializer final SerializerDefinition keySerializerDef = storeDefinition.getKeySerializer(); final SerializerDefinition valueSerializerDef = storeDefinition.getValueSerializer(); SerializerFactory serializerFactory = new DefaultSerializerFactory(); @SuppressWarnings("unchecked") final Serializer<Object> keySerializer = (Serializer<Object>) serializerFactory .getSerializer(keySerializerDef); @SuppressWarnings("unchecked") final Serializer<Object> valueSerializer = (Serializer<Object>) serializerFactory .getSerializer(valueSerializerDef); // compression strategy final CompressionStrategy keyCompressionStrategy; final CompressionStrategy valueCompressionStrategy; if (keySerializerDef != null && keySerializerDef.hasCompression()) { keyCompressionStrategy = new CompressionStrategyFactory().get(keySerializerDef.getCompression()); } else { keyCompressionStrategy = null; } if (valueSerializerDef != null && valueSerializerDef.hasCompression()) { valueCompressionStrategy = new CompressionStrategyFactory() .get(valueSerializerDef.getCompression()); } else { valueCompressionStrategy = null; } if (keyCompressionStrategy == null) { out.write("KEY_COMPRESSION_STRATEGY: None\n"); } else { out.write("KEY_COMPRESSION_STRATEGY: " + keyCompressionStrategy.getType() + "\n"); } out.write("KEY_SERIALIZER_NAME: " + keySerializerDef.getName() + "\n"); for (Map.Entry<Integer, String> entry : keySerializerDef.getAllSchemaInfoVersions().entrySet()) { out.write(String.format("KEY_SCHEMA VERSION=%d\n", entry.getKey())); out.write("====================================\n"); out.write(entry.getValue()); out.write("\n====================================\n"); } out.write("\n"); if (valueCompressionStrategy == null) { out.write("VALUE_COMPRESSION_STRATEGY: None\n"); } else { out.write("VALUE_COMPRESSION_STRATEGY: " + valueCompressionStrategy.getType() + "\n"); } out.write("VALUE_SERIALIZER_NAME: " + valueSerializerDef.getName() + "\n"); for (Map.Entry<Integer, String> entry : valueSerializerDef.getAllSchemaInfoVersions().entrySet()) { out.write(String.format("VALUE_SCHEMA %d\n", entry.getKey())); out.write("====================================\n"); out.write(entry.getValue()); out.write("\n====================================\n"); } out.write("\n"); // although the streamingOps support multiple keys, we only query // one // key here ByteArray key; try { if (keyFormat.equals("readable")) { Object keyObject; String keySerializerName = keySerializerDef.getName(); if (isAvroSchema(keySerializerName)) { Schema keySchema = Schema.parse(keySerializerDef.getCurrentSchemaInfo()); JsonDecoder decoder = new JsonDecoder(keySchema, keyString); GenericDatumReader<Object> datumReader = new GenericDatumReader<Object>(keySchema); keyObject = datumReader.read(null, decoder); } else if (keySerializerName.equals(DefaultSerializerFactory.JSON_SERIALIZER_TYPE_NAME)) { JsonReader jsonReader = new JsonReader(new StringReader(keyString)); keyObject = jsonReader.read(); } else { keyObject = keyString; } key = new ByteArray(keySerializer.toBytes(keyObject)); } else { key = new ByteArray(ByteUtils.fromHexString(keyString)); } } catch (SerializationException se) { System.err.println("Error serializing key " + keyString); System.err.println( "If this is a JSON key, you need to include escaped quotation marks in the command line if it is a string"); se.printStackTrace(); return; } catch (DecoderException de) { System.err.println("Error decoding key " + keyString); de.printStackTrace(); return; } catch (IOException io) { System.err.println("Error parsing avro string " + keyString); io.printStackTrace(); return; } boolean printedKey = false; for (final Integer queryNodeId : queryingNodes) { Iterator<QueryKeyResult> iterator; iterator = adminClient.streamingOps.queryKeys(queryNodeId, storeName, Arrays.asList(key).iterator()); final StringWriter stringWriter = new StringWriter(); QueryKeyResult queryKeyResult = iterator.next(); // de-serialize and write key byte[] keyBytes = queryKeyResult.getKey().get(); Object keyObject = keySerializer.toObject( (null == keyCompressionStrategy) ? keyBytes : keyCompressionStrategy.inflate(keyBytes)); if (!printedKey) { out.write("KEY_BYTES\n====================================\n"); out.write(queryKeyResult.getKey().toString()); out.write("\n====================================\n"); out.write("KEY_TEXT\n====================================\n"); if (keyObject instanceof GenericRecord) { out.write(keyObject.toString()); } else { new JsonFactory(new ObjectMapper()).createJsonGenerator(out).writeObject(keyObject); } out.write("\n====================================\n\n"); printedKey = true; } out.write(String.format("\nQueried node %d on store %s\n", queryNodeId, storeName)); // iterate through, de-serialize and write values if (queryKeyResult.hasValues() && queryKeyResult.getValues().size() > 0) { int versionCount = 0; out.write("VALUE " + versionCount + "\n"); for (Versioned<byte[]> versioned : queryKeyResult.getValues()) { // write version VectorClock version = (VectorClock) versioned.getVersion(); out.write("VECTOR_CLOCK_BYTE: " + ByteUtils.toHexString(version.toBytes()) + "\n"); out.write("VECTOR_CLOCK_TEXT: " + version.toString() + '[' + new Date(version.getTimestamp()).toString() + "]\n"); // write value byte[] valueBytes = versioned.getValue(); out.write("VALUE_BYTE\n====================================\n"); out.write(ByteUtils.toHexString(valueBytes)); out.write("\n====================================\n"); out.write("VALUE_TEXT\n====================================\n"); Object valueObject = valueSerializer .toObject((null == valueCompressionStrategy) ? valueBytes : valueCompressionStrategy.inflate(valueBytes)); if (valueObject instanceof GenericRecord) { out.write(valueObject.toString()); } else { new JsonFactory(new ObjectMapper()).createJsonGenerator(out).writeObject(valueObject); } out.write("\n====================================\n"); versionCount++; } } else { out.write("VALUE_RESPONSE\n====================================\n"); // write null or exception if (queryKeyResult.hasException()) { out.write(queryKeyResult.getException().toString()); } else { out.write("null"); } out.write("\n====================================\n"); } out.flush(); } } } private static void executeShowRoutingPlan(AdminClient adminClient, String storeName, List<String> keyList) throws DecoderException { Cluster cluster = adminClient.getAdminClientCluster(); List<StoreDefinition> storeDefs = adminClient.metadataMgmtOps.getRemoteStoreDefList(0).getValue(); StoreDefinition storeDef = StoreDefinitionUtils.getStoreDefinitionWithName(storeDefs, storeName); StoreRoutingPlan routingPlan = new StoreRoutingPlan(cluster, storeDef); BaseStoreRoutingPlan bRoutingPlan = new BaseStoreRoutingPlan(cluster, storeDef); final int COLUMN_WIDTH = 30; for (String keyStr : keyList) { byte[] key = ByteUtils.fromHexString(keyStr); System.out.println("Key :" + keyStr); System.out.println("Replicating Partitions :" + routingPlan.getReplicatingPartitionList(key)); System.out.println("Replicating Nodes :"); List<Integer> nodeList = routingPlan.getReplicationNodeList(routingPlan.getMasterPartitionId(key)); for (int i = 0; i < nodeList.size(); i++) { System.out.println(nodeList.get(i) + "\t" + cluster.getNodeById(nodeList.get(i)).getHost()); } System.out.println("Zone Nary information :"); HashMap<Integer, Integer> zoneRepMap = storeDef.getZoneReplicationFactor(); for (Zone zone : cluster.getZones()) { System.out.println("\tZone #" + zone.getId()); int numReplicas = -1; if (zoneRepMap == null) { // non zoned cluster numReplicas = storeDef.getReplicationFactor(); } else { // zoned cluster if (!zoneRepMap.containsKey(zone.getId())) { Utils.croak("Repfactor for Zone " + zone.getId() + " not found in storedef"); } numReplicas = zoneRepMap.get(zone.getId()); } System.out.format("%s%s%s\n", Utils.paddedString("REPLICA#", COLUMN_WIDTH), Utils.paddedString("PARTITION", COLUMN_WIDTH), Utils.paddedString("NODE", COLUMN_WIDTH)); for (int i = 0; i < numReplicas; i++) { Integer nodeId = bRoutingPlan.getNodeIdForZoneNary(zone.getId(), i, key); Integer partitionId = routingPlan.getNodesPartitionIdForKey(nodeId, key); System.out.format("%s%s%s\n", Utils.paddedString(i + "", COLUMN_WIDTH), Utils.paddedString(partitionId.toString(), COLUMN_WIDTH), Utils.paddedString( nodeId + "(" + cluster.getNodeById(nodeId).getHost() + ")", COLUMN_WIDTH)); } System.out.println(); } System.out.println("-----------------------------------------------"); System.out.println(); } } private static boolean isAvroSchema(String serializerName) { if (serializerName.equals(DefaultSerializerFactory.AVRO_GENERIC_VERSIONED_TYPE_NAME) || serializerName.equals(DefaultSerializerFactory.AVRO_GENERIC_TYPE_NAME) || serializerName.equals(DefaultSerializerFactory.AVRO_REFLECTIVE_TYPE_NAME) || serializerName.equals(DefaultSerializerFactory.AVRO_SPECIFIC_TYPE_NAME)) { return true; } else { return false; } } private static boolean confirmMetadataUpdate(Integer nodeId, AdminClient adminClient, Object value) { List<Integer> nodeIds = Lists.newArrayList(); System.out.print("\nNew metadata: \n" + value.toString() + "\n"); System.out.print("\nAffected nodes:\n"); System.out.format( "+-------+------+---------------------------------+----------+---------+------------------+%n"); System.out.printf( "|Id |Zone |Host |SocketPort|AdminPort|NumberOfPartitions|%n"); System.out.format( "+-------+------+---------------------------------+----------+---------+------------------+%n"); if (nodeId < 0) { for (Node node : adminClient.getAdminClientCluster().getNodes()) { nodeIds.add(node.getId()); System.out.format("| %-5d | %-4d | %-31s | %-5d | %-5d | %-5d |%n", node.getId(), node.getZoneId(), node.getHost(), node.getSocketPort(), node.getAdminPort(), node.getNumberOfPartitions()); System.out.format( "+-------+------+---------------------------------+----------+---------+------------------+%n"); } } System.out.print("Do you want to proceed? [Y/N]: "); Scanner in = new Scanner(System.in); String choice = in.nextLine(); if (choice.equals("Y") || choice.equals("y")) { return true; } else if (choice.equals("N") || choice.equals("n")) { return false; } else { System.out.println("Incorrect response detected. Exiting."); return false; } } }