Java tutorial
/*************************************************************************** * Copyright (C) 2009 by H-Store Project * * Brown University * * Massachusetts Institute of Technology * * Yale University * * * * Permission is hereby granted, free of charge, to any person obtaining * * a copy of this software and associated documentation files (the * * "Software"), to deal in the Software without restriction, including * * without limitation the rights to use, copy, modify, merge, publish, * * distribute, sublicense, and/or sell copies of the Software, and to * * permit persons to whom the Software is furnished to do so, subject to * * the following conditions: * * * * The above copyright notice and this permission notice shall be * * included in all copies or substantial portions of the Software. * * * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.* * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR * * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * * OTHER DEALINGS IN THE SOFTWARE. * ***************************************************************************/ package edu.brown.workload; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Pattern; import org.apache.commons.collections15.map.ListOrderedMap; import org.apache.log4j.Logger; import org.json.JSONObject; import org.voltdb.VoltTable; import org.voltdb.VoltTableRow; import org.voltdb.VoltType; import org.voltdb.WorkloadTrace; import org.voltdb.catalog.Catalog; import org.voltdb.catalog.Column; import org.voltdb.catalog.Database; import org.voltdb.catalog.Procedure; import org.voltdb.catalog.Statement; import org.voltdb.catalog.Table; import org.voltdb.types.QueryType; import edu.brown.catalog.CatalogKey; import edu.brown.catalog.CatalogUtil; import edu.brown.logging.LoggerUtil; import edu.brown.logging.LoggerUtil.LoggerBoolean; import edu.brown.statistics.Histogram; import edu.brown.statistics.ObjectHistogram; import edu.brown.statistics.TableStatistics; import edu.brown.statistics.WorkloadStatistics; import edu.brown.utils.CollectionUtil; import edu.brown.utils.StringUtil; import edu.brown.utils.ThreadUtil; import edu.brown.workload.filters.Filter; import edu.brown.workload.filters.ProcedureNameFilter; /** * * @author Andy Pavlo <pavlo@cs.brown.edu> */ public class Workload implements WorkloadTrace, Iterable<TransactionTrace> { private static final Logger LOG = Logger.getLogger(Workload.class); private static final LoggerBoolean debug = new LoggerBoolean(); private static final LoggerBoolean trace = new LoggerBoolean(); static { LoggerUtil.attachObserver(LOG, debug, trace); } public static boolean ENABLE_SHUTDOWN_HOOKS = true; /** The output stream that we're going to write our traces to **/ private FileOutputStream out; /** The last file that we loaded from **/ private File input_path; private File output_path; /** Stats Path **/ protected File stats_output; protected boolean saved_stats = false; /** Basic data members **/ private Catalog catalog; private Database catalog_db; // The following data structures are specific to Transactions private final transient ListOrderedMap<Long, TransactionTrace> txn_traces = new ListOrderedMap<Long, TransactionTrace>(); // Reverse mapping from QueryTraces to TxnIds private final transient Map<QueryTrace, Long> query_txn_xref = new ConcurrentHashMap<QueryTrace, Long>(); private transient int query_ctr = 0; // List of running queries in each batch per transaction // TXN_ID -> <BATCHID, COUNTER> private final transient Map<Long, Map<Integer, AtomicInteger>> xact_open_queries = new ConcurrentHashMap<Long, Map<Integer, AtomicInteger>>(); // A mapping from a particular procedure to a list // Procedure Catalog Key -> List of Txns protected final transient Map<String, List<TransactionTrace>> proc_xact_xref = new HashMap<String, List<TransactionTrace>>(); // We can have a list of Procedure names that we want to ignore, which we make transparent // to whomever is calling us. But this means that we need to keep track of transaction ids // that are being ignored. protected Set<String> ignored_procedures = null; protected final Set<Long> ignored_xact_ids = new HashSet<Long>(); // We also have a list of procedures that are used for bulk loading so that we can determine // the number of tuples and other various information about the tables. protected Set<String> bulkload_procedures = new HashSet<String>(); // Table Statistics (generated from bulk loading) protected WorkloadStatistics stats; protected Database stats_catalog_db; protected final Map<String, Statement> stats_load_stmts = new HashMap<String, Statement>(); // Histogram for the distribution of procedures protected final ObjectHistogram<String> proc_histogram = new ObjectHistogram<String>(); // Transaction Start Timestamp Ranges protected transient Long min_start_timestamp; protected transient Long max_start_timestamp; // ---------------------------------------------------------- // THREADS // ---------------------------------------------------------- private static WorkloadUtil.WriteThread writerThread = null; public static void writeTransactionToStream(Database catalog_db, TransactionTrace xact, OutputStream output) { writeTransactionToStream(catalog_db, xact, output, false); } public static void writeTransactionToStream(Database catalog_db, TransactionTrace xact, OutputStream output, boolean multithreaded) { if (multithreaded) { if (writerThread == null) { synchronized (Workload.class) { if (writerThread == null) { writerThread = new WorkloadUtil.WriteThread(catalog_db, output); Thread t = new Thread(writerThread); t.setDaemon(true); t.start(); } } // SYNCH } writerThread.traces.offer(xact); } else { synchronized (Workload.class) { WorkloadUtil.WriteThread.write(catalog_db, xact, output); } // SYNCH } } // ---------------------------------------------------------- // FILTERS // ---------------------------------------------------------- /** * WorkloadIterator */ public class WorkloadIterator implements Iterator<TransactionTrace> { private int idx = 0; private boolean is_init = false; private TransactionTrace peek; private final Filter filter; public WorkloadIterator(Filter filter) { this.filter = filter; } public WorkloadIterator() { this(null); } private void init() { assert (!this.is_init); this.next(); this.is_init = true; } @Override public boolean hasNext() { // Important! If this is the first time, then we need to get the next element // using next() so that we know we can even begin grabbing stuff if (!this.is_init) this.init(); return (this.peek != null); } @Override public TransactionTrace next() { // Always set the current one what we peeked ahead and saw earlier TransactionTrace current = this.peek; this.peek = null; // Now figure out what the next element should be the next time next() is called int size = Workload.this.txn_traces.size(); while (this.peek == null && this.idx++ < size) { Long next_id = Workload.this.txn_traces.get(this.idx - 1); TransactionTrace element = Workload.this.txn_traces.get(next_id); if (element == null) { LOG.warn("idx: " + this.idx); LOG.warn("next_id: " + next_id); // System.err.println("elements: " + Workload.this.element_id_xref); LOG.warn("size: " + Workload.this.txn_traces.size()); } if (this.filter == null || (this.filter != null && this.filter.apply(element) == Filter.FilterResult.ALLOW)) { this.peek = element; } } // WHILE return (current); } @Override public void remove() { // Not implemented... } }; /** * Default Constructor */ public Workload() { // Create a shutdown hook to make sure that always call finalize() if (ENABLE_SHUTDOWN_HOOKS) { if (debug.val) LOG.debug("Created shutdown hook for " + Workload.class.getName()); Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { try { Workload.this.finalize(); } catch (Throwable ex) { ex.printStackTrace(); } return; } }); } } /** * The real constructor * @param catalog */ public Workload(Catalog catalog) { this(); this.catalog = catalog; } /** * Copy Constructor! * @param catalog * @param workload */ public Workload(Catalog catalog, Workload... workloads) { this(catalog, null, workloads); } /** * Copy Constructor with a Filter! * @param catalog * @param filter * @param workload */ public Workload(Catalog catalog, Filter filter, Workload... workloads) { this(catalog); Database catalog_db = CatalogUtil.getDatabase(this.catalog); for (Workload w : workloads) { assert (w != null); for (TransactionTrace txn : CollectionUtil.iterable(w.iterator(filter))) { this.addTransaction(txn.getCatalogItem(catalog_db), txn, false); } // WHILE } // FOR } /** * We want to dump out index structures and other things to the output file * when we are going down. */ @Override protected void finalize() throws Throwable { if (this.out != null) { System.err.println("Flushing workload trace output and closing files..."); // Workload Statistics if (this.stats != null) this.saveStats(); // This was here in case I wanted to dump out auxillary data structures // As of right now, I don't need to do that, so we'll just flush and close the file this.out.flush(); this.out.close(); } super.finalize(); } public void flush() throws IOException { if (this.out != null) { this.out.flush(); } } /** * */ @Override public void setOutputPath(File path) { this.output_path = path; try { this.out = new FileOutputStream(path); if (debug.val) LOG.debug("Opened file '" + path + "' for logging workload trace"); } catch (Exception ex) { LOG.fatal("Failed to open trace output file: " + path); ex.printStackTrace(); System.exit(1); } } public void setStatsOutputPath(File path) { this.stats_output = path; } /** * * @throws Exception */ private void saveStats() throws Exception { for (TableStatistics table_stats : this.stats.getTableStatistics()) { table_stats.postprocess(this.stats_catalog_db); } // FOR if (this.stats_output != null) { this.stats.save(this.stats_output); } } /** * * @param input_path * @param catalog_db * @throws Exception */ public Workload load(File input_path, Database catalog_db) throws Exception { return this.load(input_path, catalog_db, null); } /** * * @param input_path * @param catalog_db * @param limit * @throws Exception */ public Workload load(File input_path, Database catalog_db, Filter filter) throws Exception { if (debug.val) LOG.debug("Reading workload trace from file '" + input_path + "'"); this.input_path = input_path; long start = System.currentTimeMillis(); // HACK: Throw out traces unless they have the procedures that we're looking for Pattern temp_pattern = null; if (filter != null) { List<ProcedureNameFilter> procname_filters = filter.getFilters(ProcedureNameFilter.class); if (procname_filters.isEmpty() == false) { Set<String> names = new HashSet<String>(); for (ProcedureNameFilter f : procname_filters) { for (String name : f.getProcedureNames()) { names.add(Pattern.quote(name)); } // FOR } // FOR if (names.isEmpty() == false) { temp_pattern = Pattern.compile( String.format("\"NAME\":[\\s]*\"(%s)\"", StringUtil.join("|", names)), Pattern.CASE_INSENSITIVE); if (debug.val) { LOG.debug(String.format("Fast filter for %d procedure names", names.size())); LOG.debug("PATTERN: " + temp_pattern.pattern()); } } } } final Pattern pattern = temp_pattern; final AtomicInteger counters[] = new AtomicInteger[] { new AtomicInteger(0), // ELEMENT COUNTER new AtomicInteger(0), // TXN COUNTER new AtomicInteger(0), // QUERY COUNTER new AtomicInteger(0), // WEIGHTED TXN COUNTER new AtomicInteger(0), // WEIGHTED QUERY COUNTER }; List<Runnable> all_runnables = new ArrayList<Runnable>(); int num_threads = ThreadUtil.getMaxGlobalThreads(); // Create the reader thread first WorkloadUtil.ReadThread rt = new WorkloadUtil.ReadThread(this.input_path, pattern, num_threads); all_runnables.add(rt); // Then create all of our processing threads for (int i = 0; i < num_threads; i++) { WorkloadUtil.ProcessingThread lt = new WorkloadUtil.ProcessingThread(this, this.input_path, rt, catalog_db, filter, counters); rt.processingThreads.add(lt); all_runnables.add(lt); } // FOR if (debug.val) LOG.debug(String.format("Loading workload trace using %d ProcessThreads", rt.processingThreads.size())); ThreadUtil.runNewPool(all_runnables, all_runnables.size()); VerifyWorkload.verify(catalog_db, this); long stop = System.currentTimeMillis(); LOG.info(String.format("Loaded %d txns / %d queries from '%s' in %.1f seconds using %d threads", this.txn_traces.size(), counters[2].get(), this.input_path.getName(), (stop - start) / 1000d, num_threads)); if (counters[1].get() != counters[3].get() || counters[2].get() != counters[4].get()) { LOG.info( String.format("Weighted Workload: %d txns / %d queries", counters[3].get(), counters[4].get())); } return (this); } // ---------------------------------------------------------- // ITERATORS METHODS // ---------------------------------------------------------- /** * Creates a new iterator that allows you walk through the trace elements * one by one in the order that they were created */ /** * Creates a new iterator using a Filter that allows you to walk through trace elements * and get back just the elements that you want based on some criteria * @param filter * @return */ @Override public Iterator<TransactionTrace> iterator() { return (new Workload.WorkloadIterator()); } public Iterator<TransactionTrace> iterator(Filter filter) { return (new Workload.WorkloadIterator(filter)); } // ---------------------------------------------------------- // BASIC METHODS // ---------------------------------------------------------- /** * Adds a stored procedure to a list of procedures to ignore. * If a procedure is ignored, then startTransaction() will return a null handle * * @param name - the name of the stored procedure to ignore */ @Override public void addIgnoredProcedure(String name) { if (this.ignored_procedures == null) { synchronized (this) { if (this.ignored_procedures == null) { this.ignored_procedures = new HashSet<String>(); } } // SYNCH } this.ignored_procedures.add(name.toUpperCase()); } /** * Adds a stored procedure to a list of procedures that are used for bulk-loading data * If a procedure is ignored, then startTransaction() will return a null handle, but * the trace manager could be recording additional profile statistics * * @param name - the name of the stored procedure used for bulk loading */ public void addBulkLoadProcedure(String name) { this.bulkload_procedures.add(name.toUpperCase()); } /** * Returns the number of transactions loaded into this Workload object * @return */ public int getTransactionCount() { return (this.txn_traces.size()); } /** * Returns the number of queries loaded into this Workload object * @return */ public int getQueryCount() { return (this.query_ctr); } /** * Get the set of procedures that were invoked in this workload * @param catalog_db * @return */ public Set<Procedure> getProcedures(final Database catalog_db) { Set<Procedure> procedures = new HashSet<Procedure>(); for (String proc_key : this.proc_histogram.values()) { procedures.add(CatalogKey.getFromKey(catalog_db, proc_key, Procedure.class)); } // FOR return (procedures); } /** * Return the procedure execution histogram * The keys of the Histogram will be CatalogKeys * @return */ public Histogram<String> getProcedureHistogram() { // Install debug label mapping if (!this.proc_histogram.hasDebugLabels()) { Map<Object, String> labels = new HashMap<Object, String>(); for (Object o : this.proc_histogram.values()) { labels.put(o, CatalogKey.getNameFromKey(o.toString())); } // FOR this.proc_histogram.setDebugLabels(labels); } return (this.proc_histogram); } /** * * @return */ public Long getMinStartTimestamp() { return (this.min_start_timestamp); } /** * * @return */ public Long getMaxStartTimestamp() { return (this.max_start_timestamp); } /** * Return the time interval this txn started execution in * @param xact * @param num_intervals * @return */ public int getTimeInterval(TransactionTrace xact, int num_intervals) { long timestamp = xact.getStartTimestamp(); if (timestamp == this.max_start_timestamp) timestamp--; double ratio = (timestamp - this.min_start_timestamp) / (double) (this.max_start_timestamp - this.min_start_timestamp); int interval = (int) (num_intervals * ratio); // System.err.println("min_timestamp=" + this.min_start_timestamp); // System.err.println("max_timestamp=" + this.max_start_timestamp); // System.err.println("timestamp=" + timestamp); // System.err.println("ratio= " + ratio); // System.err.println("interval=" + interval); // System.err.println("-------"); return interval; } public ObjectHistogram<Integer> getTimeIntervalProcedureHistogram(int num_intervals) { final ObjectHistogram<Integer> h = new ObjectHistogram<Integer>(); for (TransactionTrace txn_trace : this.getTransactions()) { h.put(this.getTimeInterval(txn_trace, num_intervals)); } return (h); } public ObjectHistogram<Integer> getTimeIntervalQueryHistogram(int num_intervals) { final ObjectHistogram<Integer> h = new ObjectHistogram<Integer>(); for (TransactionTrace txn_trace : this.getTransactions()) { int interval = this.getTimeInterval(txn_trace, num_intervals); int num_queries = txn_trace.getQueryCount(); h.put(interval, num_queries); } // FOR return (h); } /** * Set the catalog for monitoring * This must be set before starting any trace operations * * @param catalog - the catalog that the trace manager will be working with */ @Override public void setCatalog(Catalog catalog) { this.catalog = catalog; } /** * Utility method to remove a transaction from the various data structures that * we have to keep track of it. * @param txn_trace the transaction handle to remove */ protected void removeTransaction(TransactionTrace txn_trace) { this.txn_traces.remove(txn_trace.getTransactionId()); this.xact_open_queries.remove(txn_trace.getTransactionId()); } /** * Add a transaction to the helper data structures that we have * @param catalog_proc * @param xact */ public void addTransaction(Procedure catalog_proc, TransactionTrace xact) { this.addTransaction(catalog_proc, xact, false); } /** * * @param catalog_proc * @param txn_trace * @param force_index_update */ protected synchronized void addTransaction(Procedure catalog_proc, TransactionTrace txn_trace, boolean force_index_update) { if (debug.val) LOG.debug(String.format("Adding new %s [numTraces=%d]", txn_trace, this.txn_traces.size())); long txn_id = txn_trace.getTransactionId(); this.txn_traces.put(txn_id, txn_trace); this.xact_open_queries.put(txn_id, new HashMap<Integer, AtomicInteger>()); String proc_key = CatalogKey.createKey(catalog_proc); if (!this.proc_xact_xref.containsKey(proc_key)) { this.proc_xact_xref.put(proc_key, new ArrayList<TransactionTrace>()); } this.proc_xact_xref.get(proc_key).add(txn_trace); this.proc_histogram.put(proc_key); if (this.min_start_timestamp == null || this.min_start_timestamp > txn_trace.getStartTimestamp()) { this.min_start_timestamp = txn_trace.getStartTimestamp(); } if (this.max_start_timestamp == null || this.max_start_timestamp < txn_trace.getStartTimestamp()) { this.max_start_timestamp = txn_trace.getStartTimestamp(); } this.query_ctr += txn_trace.getQueryCount(); } /** * Returns an ordered collection of all the transactions * @return */ public Collection<TransactionTrace> getTransactions() { return (this.txn_traces.values()); } /** * Return the proper TransactionTrace object for the given txn_id * @param txn_id * @return */ public TransactionTrace getTransaction(long txn_id) { return this.txn_traces.get(txn_id); } /** * For a given Procedure catalog object, return all the transaction traces for it * @param catalog_proc * @return */ public List<TransactionTrace> getTraces(Procedure catalog_proc) { String proc_key = CatalogKey.createKey(catalog_proc); if (this.proc_xact_xref.containsKey(proc_key)) { return (this.proc_xact_xref.get(proc_key)); } return (new ArrayList<TransactionTrace>()); } /** * * @param xact_id * @param catalog_proc * @param args * @return */ private void processBulkLoader(long xact_id, Procedure catalog_proc, Object args[]) { // Process the arguments and analyze the tuples being inserted for (int ctr = 0; ctr < args.length; ctr++) { // We only want VoltTables... if (!(args[ctr] instanceof VoltTable)) continue; VoltTable voltTable = (VoltTable) args[ctr]; // Workload Initialization if (this.stats_catalog_db == null) { this.stats_catalog_db = (Database) catalog_proc.getParent(); this.stats = new WorkloadStatistics(this.stats_catalog_db); for (Table catalog_tbl : this.stats_catalog_db.getTables()) { this.stats.getTableStatistics(catalog_tbl).preprocess(this.stats_catalog_db); } // FOR } final Table catalog_tbl = CatalogUtil.getCatalogTable(stats_catalog_db, voltTable); if (catalog_tbl == null) { LOG.fatal("Failed to find corresponding table catalog object for parameter " + ctr + " in " + catalog_proc); String msg = "Table Columns: "; for (int i = 0, cnt = voltTable.getColumnCount(); i < cnt; i++) { msg += voltTable.getColumnName(i) + " "; } LOG.fatal(msg); return; } final String table_key = CatalogKey.createKey(catalog_tbl); TableStatistics table_stats = this.stats.getTableStatistics(table_key); // Temporary Loader Procedure TransactionTrace loader_xact = new TransactionTrace(xact_id, catalog_proc, new Object[0]) { /** * We need this wrapper so that when CatalogUtil tries to figure out what * the index of partitioning parameter we can just use the column index of partitioning * attribute of the table that we are inserting into */ private final Procedure proc = new Procedure() { @Override public int getPartitionparameter() { return (catalog_tbl.getPartitioncolumn().getIndex()); } @Override public Catalog getCatalog() { return (catalog); } }; @Override public Procedure getCatalogItem(Database catalog_db) { return (proc); } }; // Nasty... // Use a temporary query trace to wrap the "insert" of each tuple Statement catalog_stmt = this.stats_load_stmts.get(table_key); if (catalog_stmt == null) { catalog_stmt = catalog_proc.getStatements().add("INSERT_" + catalog_tbl.getName()); catalog_stmt.setQuerytype(QueryType.INSERT.getValue()); this.stats_load_stmts.put(table_key, catalog_stmt); // TERRIBLE HACK! // 2011-01-25 :: Why do I need to do this?? // String stmt_key = CatalogKey.createKey(catalog_stmt); // CatalogUtil.CACHE_STATEMENT_COLUMNS_KEYS.put(stmt_key, new java.util.HashSet<String>()); // CatalogUtil.CACHE_STATEMENT_COLUMNS_KEYS.get(stmt_key).add(table_key); } QueryTrace loader_query = new QueryTrace(catalog_stmt, new Object[0], 0); loader_xact.addQuery(loader_query); // Gather column types int num_columns = voltTable.getColumnCount(); VoltType col_types[] = new VoltType[num_columns]; for (int i = 0; i < num_columns; i++) { Column catalog_col = catalog_tbl.getColumns().get(i); col_types[i] = VoltType.get((byte) catalog_col.getType()); } // FOR loader_query.params = new Object[num_columns]; int num_tuples = voltTable.getRowCount(); try { LOG.info("Processing " + num_tuples + " (sample=10) tuples for statistics on " + catalog_tbl.getName()); boolean show_progress = (num_tuples > 25000); for (int i = 0; i < num_tuples; i += 10) { if (i >= num_tuples) break; VoltTableRow tuple = voltTable.fetchRow(i); for (int j = 0; j < num_columns; j++) { loader_query.params[j] = tuple.get(j, col_types[j]); } // FOR table_stats.process(stats_catalog_db, loader_xact); if (show_progress && i > 0 && i % 10000 == 0) LOG.info(i + " tuples"); // if (i > 25000) break; } // FOR LOG.info("Processing finished for " + catalog_tbl.getName()); } catch (Exception ex) { ex.printStackTrace(); System.exit(1); } } // FOR this.ignored_xact_ids.add(xact_id); } /** * * @param txn_id * @param catalog_proc * @param args * @return * @throws Exception */ @Override public TransactionTrace startTransaction(long xact_id, Procedure catalog_proc, Object args[]) { String proc_name = catalog_proc.getName().toUpperCase(); TransactionTrace xact_handle = null; // Bulk-loader Procedure if (this.bulkload_procedures.contains(proc_name)) { this.processBulkLoader(xact_id, catalog_proc, args); } // Ignored/Sysproc Procedures else if (catalog_proc.getSystemproc() || (this.ignored_procedures != null && this.ignored_procedures.contains(proc_name))) { if (debug.val) LOG.debug("Ignoring start transaction call for procedure '" + proc_name + "'"); this.ignored_xact_ids.add(xact_id); } // Procedures we want to trace else { xact_handle = new TransactionTrace(xact_id, catalog_proc, args); this.addTransaction(catalog_proc, xact_handle, false); if (debug.val) LOG.debug(String.format("Created %s TransactionTrace with %d parameters", proc_name, args.length)); // HACK if (this.catalog_db == null) this.catalog_db = CatalogUtil.getDatabase(catalog_proc); // If this is the first non bulk-loader proc that we have seen, then // go ahead and save the stats out to a file in case we crash later on if (!this.saved_stats && this.stats != null) { try { this.saveStats(); this.saved_stats = true; } catch (Exception ex) { LOG.fatal(String.format("Failed to save stats for txn #%d", xact_id), ex); throw new RuntimeException(ex); } } } return (xact_handle); } /** * * @param txn_id */ @Override public void stopTransaction(Object xact_handle, VoltTable... result) { if (xact_handle instanceof TransactionTrace) { TransactionTrace txn_trace = (TransactionTrace) xact_handle; // Make sure we have stopped all of our queries boolean unclean = false; for (QueryTrace query : txn_trace.getQueries()) { if (!query.isStopped()) { if (debug.val) LOG.warn("Trace for '" + query + "' was not stopped before the transaction. Assuming it was aborted"); query.aborted = true; unclean = true; } } // FOR if (unclean) LOG.warn("The entries in " + txn_trace + " were not stopped cleanly before the transaction was stopped"); // Mark the txn as stopped txn_trace.stop(); if (result != null) txn_trace.setOutput(result); // Remove from internal cache data structures this.removeTransaction(txn_trace); if (debug.val) LOG.debug("Stopping trace for transaction " + txn_trace); } else { LOG.fatal("Unable to stop transaction trace: Invalid transaction handle"); } // Write the trace object out to our file if it is not null if (xact_handle != null && xact_handle instanceof TransactionTrace) { TransactionTrace xact = (TransactionTrace) xact_handle; if (this.catalog_db == null) { LOG.warn("The database catalog handle is null: " + xact); } else { if (this.out == null) { if (debug.val) LOG.warn("No output path is set. Unable to log trace information to file"); } else { writeTransactionToStream(this.catalog_db, xact, this.out); } } } return; } @Override public void abortTransaction(Object xact_handle) { if (xact_handle instanceof TransactionTrace) { TransactionTrace txn_trace = (TransactionTrace) xact_handle; // Abort any open queries for (QueryTrace query : txn_trace.getQueries()) { if (query.isStopped() == false) { query.abort(); this.query_txn_xref.remove(query); } } // FOR txn_trace.abort(); if (debug.val) LOG.debug("Aborted trace for transaction " + txn_trace); // Write the trace object out to our file if it is not null if (this.catalog_db == null) { LOG.warn("The database catalog handle is null: " + txn_trace); } else { if (this.out == null) { if (debug.val) LOG.warn("No output path is set. Unable to log trace information to file"); } else { writeTransactionToStream(this.catalog_db, txn_trace, this.out); } } } else { LOG.fatal("Unable to abort transaction trace: Invalid transaction handle"); } } /** * * @param txn_id * @param catalog_stmt * @param catalog_statement * @param args * @return * @throws Exception */ @Override public Object startQuery(Object xact_handle, Statement catalog_statement, Object args[], int batch_id) { QueryTrace query_handle = null; if (xact_handle instanceof TransactionTrace) { TransactionTrace xact = (TransactionTrace) xact_handle; long txn_id = xact.getTransactionId(); if (this.ignored_xact_ids.contains(txn_id)) return (null); Map<Integer, AtomicInteger> open_queries = this.xact_open_queries.get(txn_id); // HACK if (open_queries == null) { open_queries = new HashMap<Integer, AtomicInteger>(); this.xact_open_queries.put(txn_id, open_queries); } assert (open_queries != null) : "Starting a query before starting the txn?? [" + txn_id + "]"; query_handle = new QueryTrace(catalog_statement, args, batch_id); xact.addQuery(query_handle); this.query_txn_xref.put(query_handle, txn_id); // Make sure that there aren't still running queries in the previous batch if (batch_id > 0) { AtomicInteger last_batch_ctr = open_queries.get(batch_id - 1); if (last_batch_ctr != null && last_batch_ctr.intValue() != 0) { String msg = "Txn #" + txn_id + " is trying to start a new query in batch #" + batch_id + " but there are still " + last_batch_ctr.intValue() + " queries running in batch #" + (batch_id - 1); throw new IllegalStateException(msg); } } AtomicInteger batch_ctr = open_queries.get(batch_id); if (batch_ctr == null) { synchronized (open_queries) { batch_ctr = new AtomicInteger(0); open_queries.put(batch_id, batch_ctr); } // SYNCHRONIZED } batch_ctr.incrementAndGet(); if (debug.val) LOG.debug("Created '" + catalog_statement.getName() + "' query trace record for xact '" + txn_id + "'"); } else { LOG.fatal("Unable to create new query trace: Invalid transaction handle"); } return (query_handle); } @Override public void stopQuery(Object query_handle, VoltTable result) { if (query_handle instanceof QueryTrace) { QueryTrace query = (QueryTrace) query_handle; query.stop(); if (result != null) query.setOutput(result); // Decrement open query counter for this batch Long txn_id = this.query_txn_xref.remove(query); assert (txn_id != null) : "Unexpected QueryTrace handle that doesn't have a txn id!"; Map<Integer, AtomicInteger> m = this.xact_open_queries.get(txn_id); if (m != null) { AtomicInteger batch_ctr = m.get(query.getBatchId()); int count = batch_ctr.decrementAndGet(); assert (count >= 0) : "Invalid open query counter for batch #" + query.getBatchId() + " in Txn #" + txn_id; if (debug.val) LOG.debug("Stopping trace for query " + query); } else { LOG.warn(String.format("No open query counters for txn #%d???", txn_id)); } } else { LOG.fatal("Unable to stop query trace: Invalid query handle"); } return; } public void save(File path, Database catalog_db) { this.setOutputPath(path); this.save(catalog_db); } public void save(Database catalog_db) { LOG.info("Writing out workload trace to '" + this.output_path + "'"); for (AbstractTraceElement<?> element : this) { if (element instanceof TransactionTrace) { TransactionTrace xact = (TransactionTrace) element; try { //String json = xact.toJSONString(catalog_db); //JSONObject jsonObject = new JSONObject(json); //this.out.write(jsonObject.toString(2).getBytes()); this.out.write(xact.toJSONString(catalog_db).getBytes()); this.out.write("\n".getBytes()); this.out.flush(); if (debug.val) LOG.debug("Wrote out new trace record for " + xact + " with " + xact.getQueries().size() + " queries"); } catch (Exception ex) { LOG.fatal(ex.getMessage()); ex.printStackTrace(); } } } // FOR } /** * * @param catalog_db * @return */ public String debug(Database catalog_db) { StringBuilder builder = new StringBuilder(); for (TransactionTrace xact : this.txn_traces.values()) { builder.append(xact.toJSONString(catalog_db)); } // FOR JSONObject jsonObject = null; String ret = null; try { jsonObject = new JSONObject(builder.toString()); ret = jsonObject.toString(2); } catch (Exception ex) { ex.printStackTrace(); System.exit(1); } return (ret); } }