Java tutorial
/* * Copyright 2008-2014 LinkedIn, Inc * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of * the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. */ package voldemort.tools; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.FileOutputStream; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.io.PrintStream; import java.io.PrintWriter; import java.io.StringWriter; import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import joptsimple.OptionException; import joptsimple.OptionParser; import joptsimple.OptionSet; import org.apache.commons.lang.mutable.MutableInt; import org.apache.log4j.Logger; import voldemort.VoldemortClientShell; import voldemort.client.ClientConfig; import voldemort.client.DefaultStoreClient; import voldemort.client.SocketStoreClientFactory; import voldemort.client.StoreClient; import voldemort.client.protocol.RequestFormatType; import voldemort.client.protocol.admin.AdminClient; import voldemort.cluster.Node; import voldemort.cluster.Zone; import voldemort.serialization.DefaultSerializerFactory; import voldemort.serialization.Serializer; import voldemort.serialization.SerializerDefinition; import voldemort.serialization.SerializerFactory; import voldemort.store.StoreDefinition; import voldemort.store.StoreUtils; import voldemort.utils.ByteArray; import voldemort.utils.Utils; import voldemort.versioning.VectorClock; import voldemort.versioning.Version; import voldemort.versioning.Versioned; /** * This tool reads from a keyfile and deletes * the key from the supplied stores. The tools are considered to be in * human readable format and conversion will be attempted to the * appropriate key. * * First of all understand that * * The tool also supports the following options * 1) --delete-all-versions. If you have more than one value with * conflicting versions, the tool will fail, because it may not have the * value schema to de-serialize the value and resolve the conflict. The * conflict resolution needs to happen before the key is deleted. * 2) --nodeid <> --admin-url <>. If you want to delete keys only from a * particular node. Use the above options. It is useful when you delete the * keys and if a node went down, you want to rerun the tool with that * option. * 3) --find-keys-exist <> . After the delete you can run with this * option to find if any of the keys exist. If the keys are found the tool * dumps the version of each of the keys. The tool waits for the number of * keys from each store before it completes. * * The tool creates the following files in the same directory as it is being run * <storename>_errors.txt - This contain the key and the exception and additional comments * <storename>_failure.txt - This contain just the failed keys. You can rename this file and * pass it as input for the next run. if you dont rename it the file will get overwritten * the next time which will become harder to debug. * <storename>_success.txt - The keys for which the call succeeded. It includes the missing keys. * <storename>_missing.txt - The keys which are not found in the store. * <storename>_find_keys.txt - If invoked with find-keys-exist option will dump the keys and versions * <storename>_skipped.txt - if invoked with nodeid option will dump the skipped keys. * * The tool creates one single file called status.txt * The status.txt periodically dumps some statistics about each of the stores, this may or may not be * interesting to you. */ public class DeleteKeysCLI { private static final Logger logger = Logger.getLogger(ZoneClipperCLI.class); private static OptionParser setupParser() { OptionParser parser = new OptionParser(); parser.accepts("help", "Print usage information").withOptionalArg(); parser.accepts("url", "bootstrapUrl").withRequiredArg().describedAs("bootstrap url"); parser.accepts("zone", "Zone id").withRequiredArg().describedAs("zone id").ofType(Integer.class) .defaultsTo(-1); parser.accepts("stores", "store").withRequiredArg().describedAs("stores to delete the key/value from") .withValuesSeparatedBy(',').ofType(String.class); parser.accepts("keyfile", "key file").withRequiredArg() .describedAs("file with keys to be deleted are stored as one per line"); parser.accepts("qps", "keys to be deleted per store").withRequiredArg() .describedAs("number of operations allowed per second per store").ofType(Integer.class) .defaultsTo(100); parser.accepts("delete-all-versions", "Deletes all versions for a given key").withOptionalArg(); parser.accepts("nodeid", "Delete keys if it is hosted in a node").withRequiredArg() .describedAs("Delete keys belonging to a node").ofType(Integer.class).defaultsTo(-1); parser.accepts("admin-url", "admin url").withRequiredArg().describedAs("admin url"); parser.accepts("check-keys-exist", "Verify if the number of keys exist").withRequiredArg() .describedAs("Check if the given number of keys exist in the store").ofType(Integer.class) .defaultsTo(100); return parser; } private static void printUsage() { StringBuilder help = new StringBuilder(); help.append("DeleteKeysCLI\n"); help.append(" Deletes record with the given key in the keyfile from the supplied stores\n"); help.append("Options:\n"); help.append(" Required:\n"); help.append(" --url <bootstrapUrl>\n"); help.append(" --stores <Stores comma seperated>\n"); help.append(" --keyfile <Path Of teh KeyFile> \n"); help.append(" Optional:\n"); help.append(" --qps [ max number of records allowed to be deleted per second per store ]\n"); help.append( " --delete-all-versions [ Delete all versions when more than one version is found, useful if you dont have deserializer as normal delete will fail ]\n"); help.append(" --nodeid [ If you want to delete keys that belongs only to a particular node ]\n"); help.append(" --admin-url [ admin boot strap URL, required when the nodeid parameter is passed ]\n"); help.append(" --check-keys-exist [ Check if the given number of keys exist in the store ]\n"); System.out.print(help.toString()); } private static void printUsageAndDie(String errMessage) { printUsage(); Utils.croak("\n" + errMessage); } protected final SocketStoreClientFactory factory; protected final List<String> stores; protected final Map<String, StoreClient<Object, Object>> storeClients; protected final Map<String, SerializerDefinition> serializerDefs; protected final AdminClient adminClient; protected final int qps; protected final String keyFile; protected final int nodeid; protected final boolean deleteAllVersions; protected final int checkKeysCount; public DeleteKeysCLI(String url, String adminUrl, List<String> stores, String keyFile, int zoneId, int qps, int nodeid, boolean deleteAllVersions, int checkKeysCount) throws Exception { ClientConfig clientConfig = new ClientConfig().setBootstrapUrls(url).setEnableLazy(false) .setRequestFormatType(RequestFormatType.VOLDEMORT_V3).setMaxConnectionsPerNode(15); if (zoneId != Zone.DEFAULT_ZONE_ID) { clientConfig.setClientZoneId(zoneId); } this.stores = stores; if (nodeid != -1) { if (adminUrl == null || adminUrl.length() == 0) { throw new Exception("When deleting from a specific node admin URL is expected"); } adminClient = new AdminClient(adminUrl); } else { adminClient = null; } factory = new SocketStoreClientFactory(clientConfig); storeClients = new HashMap<String, StoreClient<Object, Object>>(); serializerDefs = new HashMap<String, SerializerDefinition>(); for (String store : stores) { storeClients.put(store, factory.getStoreClient(store)); StoreDefinition storeDef = StoreUtils.getStoreDef(factory.getStoreDefs(), store); serializerDefs.put(store, storeDef.getKeySerializer()); } this.qps = qps; this.nodeid = nodeid; this.deleteAllVersions = deleteAllVersions; this.keyFile = keyFile; this.checkKeysCount = checkKeysCount; } public static class DeleteKeysTask implements Runnable { private List<Version> getAllVersions(Object key) throws Exception { DefaultStoreClient<Object, Object> defaultStoreClient = (DefaultStoreClient<Object, Object>) client; List<Version> returnValue = (List<Version>) getVersionsMethod.invoke(defaultStoreClient, key); return returnValue; } protected final BufferedWriter errorWriter; protected final BufferedWriter successWriter; protected final BufferedWriter failedKeyWriter; protected final BufferedWriter missingKeyWriter; protected final BufferedWriter skipKeyWriter; protected final BufferedWriter findKeyWriter; protected final List<BufferedWriter> writers; protected final BufferedReader keyFileReader; protected boolean isComplete = false; protected final StoreClient<Object, Object> client; protected final AdminClient adminClient; protected final SerializerDefinition serializerDef; protected final String store; protected final int qps; protected final boolean deleteAllVersions; protected final PrintStream parseErrorStream; protected final int nodeid; protected final int checkKeysCount; private final Method getVersionsMethod; private final Serializer<Object> keySerializer; final static long millisInSeconds = TimeUnit.SECONDS.toMillis(1); long keysProcessed = 0; long totalProcessed = 0; long versionCallsMade = 0; long totalVersions = 0; // check Keys statistics long totalKeysFound = 0; long minTimeStamp = 0; long maxTimeStamp = 0; boolean isTimeStampInitialized = false; private void updateKeyCounters(Version v) { if (!(v instanceof VectorClock)) throw new IllegalArgumentException("Cannot compare Versions of different types."); VectorClock vClock = (VectorClock) v; long timeStamp = vClock.getTimestamp(); if (isTimeStampInitialized == false) { isTimeStampInitialized = true; minTimeStamp = timeStamp; maxTimeStamp = timeStamp; } else { minTimeStamp = Math.min(minTimeStamp, timeStamp); maxTimeStamp = Math.max(maxTimeStamp, timeStamp); } } public boolean isComplete() { return isComplete; } public String getStatus() { String message = "Store " + store + " processed Keys " + keysProcessed + " actions processed " + totalProcessed; if (versionCallsMade > 0 || totalVersions > 0) { message += " Version Calls made " + versionCallsMade + " total versions returned " + totalVersions; } if (this.totalKeysFound > 0) { message += " Found Keys " + this.totalKeysFound; } if (isTimeStampInitialized) { message += " Min TimeStamp " + new Date(minTimeStamp) + " Max Timestamp " + new Date(maxTimeStamp); } if (isComplete) { message += " ... completed"; } return message + "\n"; } private boolean shouldProcessKey(Object key) { if (nodeid == -1) { return true; } List<Node> nodes = client.getResponsibleNodes(key); for (Node node : nodes) { if (node.getId() == nodeid) { return true; } } return false; } public DeleteKeysTask(String store, StoreClient<Object, Object> client, AdminClient adminClient, SerializerDefinition serializerDef, String keyFile, int qps, int nodeid, boolean deleteAllVersions, int checkKeysCount) throws Exception { this.store = store; this.client = client; this.serializerDef = serializerDef; this.keyFileReader = new BufferedReader(new FileReader(keyFile)); this.qps = qps; this.nodeid = nodeid; this.adminClient = adminClient; this.deleteAllVersions = deleteAllVersions; this.checkKeysCount = checkKeysCount; DefaultStoreClient<Object, Object> defClient = (DefaultStoreClient<Object, Object>) client; this.getVersionsMethod = DefaultStoreClient.class.getDeclaredMethod("getVersions", Object.class); getVersionsMethod.setAccessible(true); this.errorWriter = new BufferedWriter(new FileWriter(store + "_errors.txt", true)); this.successWriter = new BufferedWriter(new FileWriter(store + "_success.txt", true)); this.failedKeyWriter = new BufferedWriter(new FileWriter(store + "_failure.txt", true)); this.missingKeyWriter = new BufferedWriter(new FileWriter(store + "_missing.txt", true)); writers = new ArrayList<BufferedWriter>( Arrays.asList(errorWriter, successWriter, failedKeyWriter, missingKeyWriter)); if (this.nodeid != -1) { this.skipKeyWriter = new BufferedWriter(new FileWriter(store + "_skipped.txt", true)); writers.add(skipKeyWriter); SerializerFactory serializerFactory = new DefaultSerializerFactory(); this.keySerializer = (Serializer<Object>) serializerFactory.getSerializer(serializerDef); } else { this.skipKeyWriter = null; this.keySerializer = null; } if (this.checkKeysCount > 0) { findKeyWriter = new BufferedWriter(new FileWriter(store + "_find_keys.txt", true)); writers.add(findKeyWriter); } else { findKeyWriter = null; } this.parseErrorStream = new PrintStream(new FileOutputStream(store + "_parseErrors.txt", true)); } private void flushStreams() throws IOException { for (BufferedWriter writer : writers) { writer.flush(); } parseErrorStream.flush(); } private void closeStreams() throws IOException { for (BufferedWriter writer : writers) { writer.close(); } parseErrorStream.close(); keyFileReader.close(); } private void writeKeyResult(String keyStr, boolean isKeyPresent) throws IOException { successWriter.write(keyStr + "\n"); if (isKeyPresent == false) { missingKeyWriter.write(keyStr + "\n"); } } private static void outputError(BufferedWriter failedKeys, BufferedWriter errorWriter, String errorMessage, Exception e, String key) throws IOException { failedKeys.write(key + "\n"); errorWriter.write(key + "\n"); errorWriter.write("Reason for failure " + errorMessage + "\n"); if (e != null) { errorWriter.write("Exception Message " + e.getMessage() + "\n"); StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); e.printStackTrace(pw); errorWriter.write("Exception stack " + sw.toString() + "\n"); } } public void run() { try { internalRun(); } catch (IOException e) { System.out.println( "Store processing interrupted " + store + " because of IOException " + e.getMessage()); e.printStackTrace(); } } public void internalRun() throws IOException { long currentQps = 0; final long BATCH_SIZE = 1000; long nextProcessed = BATCH_SIZE; String keyStr; long totalTimeProcessed = 0; long currentTimeMS = System.currentTimeMillis(); while ((keyStr = keyFileReader.readLine()) != null) { if (keyStr.length() == 0) { System.out.println("Skipping empty line"); continue; } if (this.checkKeysCount > 0 && totalKeysFound >= this.checkKeysCount) { break; } MutableInt nextParsePos = new MutableInt(0); Object key = VoldemortClientShell.parseObject(serializerDef, keyStr, nextParsePos, parseErrorStream); if (key == null) { outputError(failedKeyWriter, errorWriter, "Failed to parse Key ####", null, keyStr); continue; } keysProcessed++; if (shouldProcessKey(key) == false) { skipKeyWriter.write(keyStr + "\n"); continue; } if (currentQps >= this.qps) { long elapsedTime = System.currentTimeMillis() - currentTimeMS; currentQps = 0; totalTimeProcessed += elapsedTime; if (elapsedTime <= millisInSeconds) { long remainingMillis = millisInSeconds - elapsedTime; try { Thread.sleep(remainingMillis); } catch (InterruptedException e) { e.printStackTrace(); } } currentTimeMS = System.currentTimeMillis(); } if (totalProcessed >= nextProcessed) { System.out.println("Store " + store + " Processed record count " + keysProcessed + " elapsed time " + totalTimeProcessed + " qps " + qps); nextProcessed += BATCH_SIZE; flushStreams(); } currentQps++; totalProcessed++; try { List<Version> versions = null; boolean hasFetchedVersions = false; if (this.nodeid != -1) { hasFetchedVersions = true; List<Versioned<byte[]>> versionedValues = adminClient.storeOps.getNodeKey(this.store, this.nodeid, new ByteArray(this.keySerializer.toBytes(key))); versionCallsMade++; if (versionedValues != null && versionedValues.size() > 0) { versions = new ArrayList<Version>(); for (Versioned<byte[]> versionedValue : versionedValues) { versions.add(versionedValue.getVersion()); } } } else if (this.deleteAllVersions == true || this.checkKeysCount > 0) { hasFetchedVersions = true; versions = getAllVersions(key); versionCallsMade++; } if (hasFetchedVersions) { if (versions != null && versions.size() > 0) { totalVersions += versions.size(); } else { // It fetched versions but nothing was returned, so loop around writeKeyResult(keyStr, false); continue; } } boolean isKeyPresent = true; if (this.checkKeysCount > 0) { for (Version v : versions) { updateKeyCounters(v); String message = "Key " + keyStr + " Version " + v + "\n"; findKeyWriter.write(message); } totalKeysFound++; } else if (hasFetchedVersions) { for (Version v : versions) { boolean result = client.delete(key, v); isKeyPresent = isKeyPresent && result; } } else { isKeyPresent = client.delete(key); } writeKeyResult(keyStr, isKeyPresent); } catch (Exception e) { outputError(failedKeyWriter, errorWriter, "Error during client delete operation", e, keyStr); } } isComplete = true; closeStreams(); } } public void deleteKeys() throws Exception { Map<String, Thread> workers = new HashMap<String, Thread>(); Map<String, DeleteKeysTask> tasks = new HashMap<String, DeleteKeysTask>(); BufferedWriter statusWriter = new BufferedWriter(new FileWriter("status.txt", true)); for (String store : this.stores) { DeleteKeysTask task = new DeleteKeysTask(store, storeClients.get(store), this.adminClient, serializerDefs.get(store), this.keyFile, this.qps, this.nodeid, this.deleteAllVersions, this.checkKeysCount); Thread worker = new Thread(task); workers.put(store, worker); tasks.put(store, task); statusWriter.write("Created thread " + worker.getId() + " for store " + store + "\n"); System.out.println("Created thread " + worker.getId() + " for store " + store); worker.start(); } statusWriter.flush(); boolean isAllCompleted = false; while (isAllCompleted == false) { isAllCompleted = true; for (String store : stores) { Thread worker = workers.get(store); DeleteKeysTask task = tasks.get(store); System.out.print(task.getStatus()); statusWriter.write(task.getStatus()); if (task.isComplete()) { continue; } if (worker.isAlive() == false) { System.out.println("Thread processing it has died " + store); statusWriter.write("Thread processing it has died " + store + "\n"); continue; } isAllCompleted = false; } statusWriter.flush(); if (isAllCompleted == false) { Thread.sleep(1000 * 15); } } statusWriter.close(); } public static void main(String[] args) throws Exception { OptionParser parser = null; OptionSet options = null; try { parser = setupParser(); options = parser.parse(args); } catch (OptionException oe) { parser.printHelpOn(System.out); printUsageAndDie("Exception when parsing arguments : " + oe.getMessage()); return; } /* validate options */ if (options.has("help")) { printUsage(); return; } if (!options.hasArgument("url") || !options.hasArgument("stores") || !options.hasArgument("keyfile")) { printUsageAndDie("Missing a required argument."); return; } boolean deleteAllVersions = options.has("delete-all-versions"); System.out.println("New Delete All versions value " + deleteAllVersions); String url = (String) options.valueOf("url"); String keyFile = (String) options.valueOf("keyfile"); keyFile.replace("~", System.getProperty("user.home")); int zoneId = Zone.DEFAULT_ZONE_ID; if (options.hasArgument("zone")) { zoneId = ((Integer) options.valueOf("zone")).intValue(); } int qps = ((Integer) options.valueOf("qps")).intValue(); int nodeid = ((Integer) options.valueOf("nodeid")).intValue(); String adminUrl = ""; if (nodeid != -1) { adminUrl = (String) options.valueOf("admin-url"); } List<String> stores = null; if (options.hasArgument("stores")) { @SuppressWarnings("unchecked") List<String> list = (List<String>) options.valuesOf("stores"); stores = list; } int checkKeysCount = 0; if (options.hasArgument("check-keys-exist")) { checkKeysCount = ((Integer) options.valueOf("check-keys-exist")).intValue(); if (checkKeysCount <= 0) { throw new Exception("Expect a positive value for check-keys-exist"); } } DeleteKeysCLI deleteKeysCLI = new DeleteKeysCLI(url, adminUrl, stores, keyFile, zoneId, qps, nodeid, deleteAllVersions, checkKeysCount); deleteKeysCLI.deleteKeys(); } }