Java tutorial
/*************************************************************************** * Copyright (C) 2009 by H-Store Project * * Brown University * * Massachusetts Institute of Technology * * Yale University * * * * Permission is hereby granted, free of charge, to any person obtaining * * a copy of this software and associated documentation files (the * * "Software"), to deal in the Software without restriction, including * * without limitation the rights to use, copy, modify, merge, publish, * * distribute, sublicense, and/or sell copies of the Software, and to * * permit persons to whom the Software is furnished to do so, subject to * * the following conditions: * * * * The above copyright notice and this permission notice shall be * * included in all copies or substantial portions of the Software. * * * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.* * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR * * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * * OTHER DEALINGS IN THE SOFTWARE. * ***************************************************************************/ package edu.brown.utils; import java.io.*; import java.lang.reflect.Constructor; import java.lang.reflect.Field; import java.text.ParseException; import java.util.*; import java.util.Map.Entry; import java.util.regex.Pattern; import org.apache.commons.collections15.map.ListOrderedMap; import org.apache.log4j.Logger; import org.voltdb.CatalogContext; import org.voltdb.VoltType; import org.voltdb.catalog.*; import org.voltdb.utils.JarReader; import org.voltdb.utils.VoltTypeUtil; import edu.brown.benchmark.AbstractProjectBuilder; import edu.brown.catalog.CatalogUtil; import edu.brown.catalog.ClusterConfiguration; import edu.brown.catalog.FixCatalog; import edu.brown.costmodel.AbstractCostModel; import edu.brown.costmodel.SingleSitedCostModel; import edu.brown.costmodel.TimeIntervalCostModel; import edu.brown.designer.*; import edu.brown.designer.indexselectors.*; import edu.brown.designer.mappers.*; import edu.brown.designer.partitioners.*; import edu.brown.designer.partitioners.plan.PartitionPlan; import edu.brown.hashing.*; import edu.brown.logging.LoggerUtil; import edu.brown.mappings.ParameterMappingsSet; import edu.brown.markov.EstimationThresholds; import edu.brown.statistics.*; import edu.brown.workload.*; import edu.brown.workload.filters.*; import edu.brown.hstore.conf.HStoreConf; /** * @author pavlo */ public class ArgumentsParser { protected static final Logger LOG = Logger.getLogger(ArgumentsParser.class); static { LoggerUtil.setupLogging(); } // HACK public static boolean DISABLE_UPDATE_CATALOG = false; // -------------------------------------------------------------- // INPUT PARAMETERS // -------------------------------------------------------------- public static final String PARAM_CATALOG = "catalog"; public static final String PARAM_CATALOG_JAR = PARAM_CATALOG + ".jar"; public static final String PARAM_CATALOG_OUTPUT = PARAM_CATALOG + ".output"; public static final String PARAM_CATALOG_TYPE = PARAM_CATALOG + ".type"; public static final String PARAM_CATALOG_SCHEMA = PARAM_CATALOG + ".schema"; public static final String PARAM_CATALOG_LABELS = PARAM_CATALOG + ".labels"; public static final String PARAM_CATALOG_HOSTS = PARAM_CATALOG + ".hosts"; public static final String PARAM_CATALOG_NUM_HOSTS = PARAM_CATALOG + ".numhosts"; public static final String PARAM_CATALOG_HOST_CORES = PARAM_CATALOG + ".hosts.cores"; public static final String PARAM_CATALOG_HOST_THREADS = PARAM_CATALOG + ".hosts.threads"; public static final String PARAM_CATALOG_HOST_MEMORY = PARAM_CATALOG + ".hosts.memory"; public static final String PARAM_CATALOG_PORT = PARAM_CATALOG + ".port"; public static final String PARAM_CATALOG_PARTITION = PARAM_CATALOG + ".partition"; public static final String PARAM_CATALOG_SITES_PER_HOST = PARAM_CATALOG + ".hosts.numsites"; public static final String PARAM_CATALOG_PARTITIONS_PER_SITE = PARAM_CATALOG + ".site.numpartitions"; public static final String PARAM_CONF = "conf"; public static final String PARAM_CONF_OUTPUT = PARAM_CONF + ".output"; public static final String PARAM_WORKLOAD = "workload"; public static final String PARAM_WORKLOAD_XACT_LIMIT = PARAM_WORKLOAD + ".xactlimit"; public static final String PARAM_WORKLOAD_XACT_WEIGHTS = PARAM_WORKLOAD + ".xactweights"; public static final String PARAM_WORKLOAD_XACT_OFFSET = PARAM_WORKLOAD + ".xactoffset"; public static final String PARAM_WORKLOAD_QUERY_LIMIT = PARAM_WORKLOAD + ".querylimit"; public static final String PARAM_WORKLOAD_REMOVE_DUPES = PARAM_WORKLOAD + ".removedupes"; public static final String PARAM_WORKLOAD_PROC_EXCLUDE = PARAM_WORKLOAD + ".procexclude"; public static final String PARAM_WORKLOAD_PROC_INCLUDE = PARAM_WORKLOAD + ".procinclude"; public static final String PARAM_WORKLOAD_PROC_SAMPLE = PARAM_WORKLOAD + ".sampling"; public static final String PARAM_WORKLOAD_PROC_INCLUDE_MULTIPLIER = PARAM_WORKLOAD_PROC_INCLUDE + ".multiplier"; public static final String PARAM_WORKLOAD_RANDOM_PARTITIONS = PARAM_WORKLOAD + ".randompartitions"; public static final String PARAM_WORKLOAD_BASE_PARTITIONS = PARAM_WORKLOAD + ".basepartitions"; public static final String PARAM_WORKLOAD_OUTPUT = PARAM_WORKLOAD + ".output"; public static final String PARAM_STATS = "stats"; public static final String PARAM_STATS_OUTPUT = PARAM_STATS + ".output"; public static final String PARAM_STATS_SCALE_FACTOR = PARAM_STATS + ".scalefactor"; public static final String PARAM_MAPPINGS = "mappings"; public static final String PARAM_MAPPINGS_OUTPUT = PARAM_MAPPINGS + ".output"; public static final String PARAM_MAPPINGS_THRESHOLD = PARAM_MAPPINGS + ".threshold"; public static final String PARAM_MARKOV = "markov"; public static final String PARAM_MARKOV_OUTPUT = PARAM_MARKOV + ".output"; public static final String PARAM_MARKOV_THRESHOLDS = PARAM_MARKOV + ".thresholds"; public static final String PARAM_MARKOV_THRESHOLDS_VALUE = PARAM_MARKOV + ".thresholds.value"; public static final String PARAM_MARKOV_THRESHOLDS_OUTPUT = PARAM_MARKOV_THRESHOLDS + ".output"; public static final String PARAM_MARKOV_PARTITIONS = PARAM_MARKOV + ".partitions"; public static final String PARAM_MARKOV_TOPK = PARAM_MARKOV + ".topk"; public static final String PARAM_MARKOV_ROUNDS = PARAM_MARKOV + ".rounds"; public static final String PARAM_MARKOV_THREADS = PARAM_MARKOV + ".threads"; public static final String PARAM_MARKOV_SPLIT = PARAM_MARKOV + ".split"; public static final String PARAM_MARKOV_GLOBAL = PARAM_MARKOV + ".global"; public static final String PARAM_MARKOV_RECOMPUTE_END = PARAM_MARKOV + ".recompute_end"; public static final String PARAM_MARKOV_RECOMPUTE_WARMUP = PARAM_MARKOV + ".recompute_warmup"; public static final String PARAM_MARKOV_SPLIT_TRAINING = PARAM_MARKOV_SPLIT + ".training"; public static final String PARAM_MARKOV_SPLIT_VALIDATION = PARAM_MARKOV_SPLIT + ".validation"; public static final String PARAM_MARKOV_SPLIT_TESTING = PARAM_MARKOV_SPLIT + ".testing"; private static final String PARAM_CONFLICTS = "conflicts"; public static final String PARAM_CONFLICTS_EXCLUDE_PROCEDURES = PARAM_CONFLICTS + ".exclude_procedures"; public static final String PARAM_CONFLICTS_EXCLUDE_STATEMENTS = PARAM_CONFLICTS + ".exclude_statements"; public static final String PARAM_CONFLICTS_FOCUS_PROCEDURE = PARAM_CONFLICTS + ".focus"; public static final String PARAM_DESIGNER = "designer"; public static final String PARAM_DESIGNER_PARTITIONER = PARAM_DESIGNER + ".partitioner"; public static final String PARAM_DESIGNER_MAPPER = PARAM_DESIGNER + ".mapper"; public static final String PARAM_DESIGNER_INDEXER = PARAM_DESIGNER + ".indexer"; public static final String PARAM_DESIGNER_THREADS = PARAM_DESIGNER + ".threads"; public static final String PARAM_DESIGNER_INTERVALS = PARAM_DESIGNER + ".intervals"; public static final String PARAM_DESIGNER_COSTMODEL = PARAM_DESIGNER + ".costmodel"; public static final String PARAM_DESIGNER_HINTS = PARAM_DESIGNER + ".hints"; public static final String PARAM_DESIGNER_HINTS_PREFIX = PARAM_DESIGNER_HINTS + "."; public static final String PARAM_DESIGNER_CHECKPOINT = PARAM_DESIGNER + ".checkpoint"; public static final String PARAM_PARTITION_PLAN = "partitionplan"; public static final String PARAM_PARTITION_PLAN_OUTPUT = PARAM_PARTITION_PLAN + ".output"; public static final String PARAM_PARTITION_PLAN_APPLY = PARAM_PARTITION_PLAN + ".apply"; public static final String PARAM_PARTITION_PLAN_REMOVE_PROCS = PARAM_PARTITION_PLAN + ".removeprocs"; public static final String PARAM_PARTITION_PLAN_RANDOM_PROCS = PARAM_PARTITION_PLAN + ".randomprocs"; public static final String PARAM_PARTITION_PLAN_NO_SECONDARY = PARAM_PARTITION_PLAN + ".nosecondary"; public static final String PARAM_PARTITION_PLAN_IGNORE_MISSING = PARAM_PARTITION_PLAN + ".ignore_missing"; public static final String PARAM_PARTITION_MAP = "partitionmap"; public static final String PARAM_PARTITION_MAP_OUTPUT = PARAM_PARTITION_MAP + ".output"; private static final String PARAM_HASHER = "hasher"; public static final String PARAM_HASHER_CLASS = PARAM_HASHER + ".class"; public static final String PARAM_HASHER_PROFILE = PARAM_HASHER + ".profile"; public static final String PARAM_HASHER_OUTPUT = PARAM_HASHER + ".output"; private static final String PARAM_TERMINAL = "terminal"; public static final String PARAM_TERMINAL_CSV = PARAM_TERMINAL + ".csv"; public static final String PARAM_TERMINAL_HOST = PARAM_TERMINAL + ".host"; public static final String PARAM_TERMINAL_PORT = PARAM_TERMINAL + ".port"; private static final String PARAM_SITE = "site"; public static final String PARAM_SITE_HOST = PARAM_SITE + ".host"; public static final String PARAM_SITE_PORT = PARAM_SITE + ".port"; public static final String PARAM_SITE_PARTITION = PARAM_SITE + ".partition"; public static final String PARAM_SITE_ID = PARAM_SITE + ".id"; public static final String PARAM_SITE_IGNORE_DTXN = PARAM_SITE + ".ignore_dtxn"; public static final String PARAM_SITE_STATUS_INTERVAL = PARAM_SITE + ".statusinterval"; public static final String PARAM_SITE_STATUS_INTERVAL_KILL = PARAM_SITE + ".statusinterval_kill"; public static final String PARAM_SITE_CLEANUP_INTERVAL = PARAM_SITE + ".cleanup_interval"; public static final String PARAM_SITE_CLEANUP_TXN_EXPIRE = PARAM_SITE + ".cleanup_txn_expire"; public static final String PARAM_SITE_ENABLE_PROFILING = PARAM_SITE + ".enable_profiling"; public static final String PARAM_SITE_MISPREDICT_CRASH = PARAM_SITE + ".mispredict_crash"; public static final List<String> PARAMS = new ArrayList<String>(); static { for (Field field : ArgumentsParser.class.getDeclaredFields()) { try { if (field.getName().startsWith("PARAM_")) { ArgumentsParser.PARAMS.add(field.get(null).toString()); } } catch (Exception ex) { ex.printStackTrace(); System.exit(1); } } // FOR }; // STATIC /** * Parameter Key -> Value */ private final Map<String, String> params = new LinkedHashMap<String, String>(); private final Map<String, String> conf_params = new LinkedHashMap<String, String>(); /** * "Leftover" Parameters (getopt style) */ private final List<String> opt_params = new ArrayList<String>(); /** * Special Case: Designer Hints Override */ private final Map<String, String> hints_params = new ListOrderedMap<String, String>(); /** * Catalog Attributes */ public CatalogContext catalogContext = null; public Catalog catalog = null; public Database catalog_db = null; public File catalog_path = null; public ProjectType catalog_type = null; /** * Workload Trace Attributes */ public Workload workload = null; public File workload_path = null; public Long workload_xact_limit = null; public Long workload_xact_offset = 0l; public Long workload_query_limit = null; public final Set<Integer> workload_base_partitions = new HashSet<Integer>(); public Filter workload_filter = null; /** * Workload Statistics Attributes */ public WorkloadStatistics stats = null; public File stats_path = null; /** * Transaction Estimation Stuff */ public final ParameterMappingsSet param_mappings = new ParameterMappingsSet(); public EstimationThresholds thresholds; /** * Designer Components */ public int max_concurrent = 1; public int num_intervals = 100; public final DesignerHints designer_hints = new DesignerHints(); public File designer_checkpoint; public Class<? extends AbstractPartitioner> partitioner_class = BranchAndBoundPartitioner.class; public Class<? extends AbstractMapper> mapper_class = AffinityMapper.class; public Class<? extends AbstractIndexSelector> indexer_class = SimpleIndexSelector.class; public PartitionPlan pplan; public PartitionMapping pmap; public IndexPlan iplan; public Class<? extends AbstractCostModel> costmodel_class = SingleSitedCostModel.class; public AbstractCostModel costmodel; /** * Hasher */ public Class<? extends AbstractHasher> hasher_class = DefaultHasher.class; public AbstractHasher hasher; /** * Empty Constructor */ public ArgumentsParser() { // Nothing to do } public List<String> getOptParams() { return (this.opt_params); } public int getOptParamCount() { return (this.opt_params.size()); } public String getOptParam(int idx) { return (this.opt_params.get(idx)); } @SuppressWarnings("unchecked") public <T> T getOptParam(int idx, VoltType vt) { assert (idx >= 0); String val = (idx < this.opt_params.size() ? this.opt_params.get(idx) : null); if (val != null) { try { return ((T) VoltTypeUtil.getObjectFromString(vt, val)); } catch (ParseException ex) { throw new RuntimeException("Failed to cast optional parameter " + idx + " [value=" + val + "]", ex); } } return (null); } public Boolean getBooleanOptParam(int idx) { return (this.getOptParam(idx, VoltType.BOOLEAN)); } public Byte getByteOptParam(int idx) { Object obj = this.getOptParam(idx, VoltType.TINYINT); if (obj != null) obj = ((Number) obj).byteValue(); return ((Byte) obj); } public Short getShortOptParam(int idx) { Object obj = this.getOptParam(idx, VoltType.SMALLINT); if (obj != null) obj = ((Number) obj).shortValue(); return ((Short) obj); } public Integer getIntOptParam(int idx) { Object obj = this.getOptParam(idx, VoltType.INTEGER); if (obj != null) obj = ((Number) obj).intValue(); return ((Integer) obj); } public Long getLongOptParam(int idx) { return (this.getOptParam(idx, VoltType.BIGINT)); } public Double getDoubleOptParam(int idx) { return (this.getOptParam(idx, VoltType.DECIMAL)); } public Map<String, String> getParams() { return (this.params); } public String getParam(String key) { return (this.params.get(key)); } public Integer getIntParam(String key) { String val = this.params.get(key); Integer ret = null; if (val != null) ret = Integer.valueOf(val); return (ret); } public Long getLongParam(String key) { String val = this.params.get(key); Long ret = null; if (val != null) ret = Long.valueOf(val); return (ret); } public Double getDoubleParam(String key) { String val = this.params.get(key); Double ret = null; if (val != null) ret = Double.valueOf(val); return (ret); } public Boolean getBooleanParam(String key, Boolean defaultValue) { String val = this.params.get(key); Boolean ret = defaultValue; if (val != null) ret = Boolean.valueOf(val); return (ret); } public Boolean getBooleanParam(String key) { return (this.getBooleanParam(key, null)); } public File getFileParam(String key) { String val = this.params.get(key); File ret = null; if (val != null) ret = new File(val); return (ret); } public boolean hasParam(String key) { return (this.params.get(key) != null); } public boolean hasIntParam(String key) { if (this.hasParam(key)) { try { Long val = Long.valueOf(this.params.get(key)); if (val != null) return (true); } catch (NumberFormatException ex) { // Nothing... } } return (false); } public boolean hasDoubleParam(String key) { if (this.hasParam(key)) { try { Double val = Double.valueOf(this.params.get(key)); if (val != null) return (true); } catch (NumberFormatException ex) { // Nothing... } } return (false); } public boolean hasBooleanParam(String key) { if (this.hasParam(key)) { Boolean val = Boolean.valueOf(this.params.get(key)); if (val != null) return (true); } return (false); } public void setDatabase(Database catalog_db) { this.catalog_db = catalog_db; } /** * Check whether they have all the parameters they need * * @param params * @throws IllegalArgumentException */ public void require(String... params) throws IllegalArgumentException { for (String param : params) { if (!this.hasParam(param)) { throw new IllegalArgumentException( "Missing parameter '" + param + "'. Required Parameters = " + Arrays.asList(params)); } } // FOR return; } public Map<String, String> getHStoreConfParameters() { return (this.conf_params); } /** * Return an object with the proper objects loaded * * @param args * @return * @throws Exception */ public static ArgumentsParser load(String args[], String... required) throws Exception { ArgumentsParser au = new ArgumentsParser(); au.process(args, required); // System.out.println("catalog: " + au.catalog); // System.out.println("catalog_db: " + au.catalog_db); // System.out.println("workload: " + au.workload); // System.out.println("workload.limit: " + au.workload_limit); // System.out.println("stats: " + au.stats); return (au); } /** * @throws Exception */ private void loadWorkload() throws Exception { final boolean debug = LOG.isDebugEnabled(); // Workload Trace if (this.params.containsKey(PARAM_WORKLOAD)) { assert (this.catalog_db != null) : "Missing catalog!"; File path = new File(this.params.get(PARAM_WORKLOAD)); boolean weightedTxns = this.getBooleanParam(PARAM_WORKLOAD_XACT_WEIGHTS, false); if (debug) LOG.debug("Use Transaction Weights in Limits: " + weightedTxns); // This will prune out duplicate trace records... if (params.containsKey(PARAM_WORKLOAD_REMOVE_DUPES)) { DuplicateTraceFilter filter = new DuplicateTraceFilter(); this.workload_filter = (this.workload_filter != null ? filter.attach(this.workload_filter) : filter); if (debug) LOG.debug("Attached " + filter.debugImpl()); } // TRANSACTION OFFSET if (params.containsKey(PARAM_WORKLOAD_XACT_OFFSET)) { this.workload_xact_offset = Long.parseLong(params.get(PARAM_WORKLOAD_XACT_OFFSET)); ProcedureLimitFilter filter = new ProcedureLimitFilter(-1l, this.workload_xact_offset, weightedTxns); // Important! The offset should go in the front! this.workload_filter = (this.workload_filter != null ? filter.attach(this.workload_filter) : filter); if (debug) LOG.debug("Attached " + filter.debugImpl()); } // BASE PARTITIONS if (params.containsKey(PARAM_WORKLOAD_RANDOM_PARTITIONS) || params.containsKey(PARAM_WORKLOAD_BASE_PARTITIONS)) { BasePartitionTxnFilter filter = new BasePartitionTxnFilter( new PartitionEstimator(this.catalogContext)); // FIXED LIST if (params.containsKey(PARAM_WORKLOAD_BASE_PARTITIONS)) { for (String p_str : this.getParam(PARAM_WORKLOAD_BASE_PARTITIONS).split(",")) { workload_base_partitions.add(Integer.valueOf(p_str)); } // FOR // RANDOM } else { double factor = this.getDoubleParam(PARAM_WORKLOAD_RANDOM_PARTITIONS); List<Integer> all_partitions = new ArrayList<Integer>(catalogContext.getAllPartitionIds()); Collections.shuffle(all_partitions, new Random()); workload_base_partitions .addAll(all_partitions.subList(0, (int) (all_partitions.size() * factor))); } filter.addPartitions(workload_base_partitions); this.workload_filter = (this.workload_filter != null ? this.workload_filter.attach(filter) : filter); if (debug) LOG.debug("Attached " + filter.debugImpl()); } // Txn Limit this.workload_xact_limit = this.getLongParam(PARAM_WORKLOAD_XACT_LIMIT); ObjectHistogram<String> proc_histogram = null; // Include/exclude procedures from the traces if (params.containsKey(PARAM_WORKLOAD_PROC_INCLUDE) || params.containsKey(PARAM_WORKLOAD_PROC_EXCLUDE)) { Filter filter = new ProcedureNameFilter(weightedTxns); // INCLUDE String temp = params.get(PARAM_WORKLOAD_PROC_INCLUDE); if (temp != null && !temp.equals(ProcedureNameFilter.INCLUDE_ALL)) { // We can take the counts for PROC_INCLUDE and scale them // with the multiplier double multiplier = 1.0d; if (this.hasDoubleParam(PARAM_WORKLOAD_PROC_INCLUDE_MULTIPLIER)) { multiplier = this.getDoubleParam(PARAM_WORKLOAD_PROC_INCLUDE_MULTIPLIER); if (debug) LOG.debug("Workload Procedure Multiplier: " + multiplier); } // Default Txn Frequencies String procinclude = params.get(PARAM_WORKLOAD_PROC_INCLUDE); if (procinclude.equalsIgnoreCase("default")) { procinclude = AbstractProjectBuilder.getProjectBuilder(catalog_type) .getTransactionFrequencyString(); } Map<String, Integer> limits = new HashMap<String, Integer>(); int total_unlimited = 0; int total = 0; for (String proc_name : procinclude.split(",")) { int limit = -1; // Check if there is a limit for this procedure if (proc_name.contains(":")) { String pieces[] = proc_name.split(":"); proc_name = pieces[0]; limit = (int) Math.round(Integer.parseInt(pieces[1]) * multiplier); } if (limit < 0) { if (proc_histogram == null) { if (debug) LOG.debug("Generating procedure histogram from workload file"); proc_histogram = WorkloadUtil.getProcedureHistogram(path); } limit = (int) proc_histogram.get(proc_name, 0); total_unlimited += limit; } else { total += limit; } limits.put(proc_name, limit); } // FOR // If we have a workload limit and some txns that we want // to get unlimited // records from, then we want to modify the other txns so // that we fill in the "gap" if (this.workload_xact_limit != null && total_unlimited > 0) { int remaining = this.workload_xact_limit.intValue() - total - total_unlimited; if (remaining > 0) { for (Entry<String, Integer> e : limits.entrySet()) { double ratio = e.getValue() / (double) total; e.setValue((int) Math.ceil(e.getValue() + (ratio * remaining))); } // FOR } } ObjectHistogram<String> proc_multiplier_histogram = null; if (debug) { if (proc_histogram != null) LOG.debug("Full Workload Histogram:\n" + proc_histogram); proc_multiplier_histogram = new ObjectHistogram<String>(); } total = 0; for (Entry<String, Integer> e : limits.entrySet()) { if (debug) proc_multiplier_histogram.put(e.getKey(), e.getValue()); ((ProcedureNameFilter) filter).include(e.getKey(), e.getValue()); total += e.getValue(); } // FOR if (debug) LOG.debug("Multiplier Histogram [total=" + total + "]:\n" + proc_multiplier_histogram); } // EXCLUDE temp = params.get(PARAM_WORKLOAD_PROC_EXCLUDE); if (temp != null) { for (String proc_name : params.get(PARAM_WORKLOAD_PROC_EXCLUDE).split(",")) { ((ProcedureNameFilter) filter).exclude(proc_name); } // FOR } // Sampling!! if (this.getBooleanParam(PARAM_WORKLOAD_PROC_SAMPLE, false)) { if (debug) LOG.debug("Attaching sampling filter"); if (proc_histogram == null) proc_histogram = WorkloadUtil.getProcedureHistogram(path); Map<String, Integer> proc_includes = ((ProcedureNameFilter) filter).getProcIncludes(); SamplingFilter sampling_filter = new SamplingFilter(proc_includes, proc_histogram); filter = sampling_filter; if (debug) LOG.debug("Workload Procedure Histogram:\n" + proc_histogram); } // Attach our new filter to the chain (or make it the head if // it's the first one) this.workload_filter = (this.workload_filter != null ? this.workload_filter.attach(filter) : filter); if (debug) LOG.debug("Attached " + filter.debugImpl()); } // TRANSACTION LIMIT if (this.workload_xact_limit != null) { ProcedureLimitFilter filter = new ProcedureLimitFilter(this.workload_xact_limit, weightedTxns); this.workload_filter = (this.workload_filter != null ? this.workload_filter.attach(filter) : filter); if (debug) LOG.debug("Attached " + filter.debugImpl()); } // QUERY LIMIT if (params.containsKey(PARAM_WORKLOAD_QUERY_LIMIT)) { this.workload_query_limit = Long.parseLong(params.get(PARAM_WORKLOAD_QUERY_LIMIT)); QueryLimitFilter filter = new QueryLimitFilter(this.workload_query_limit); this.workload_filter = (this.workload_filter != null ? this.workload_filter.attach(filter) : filter); } if (this.workload_filter != null && debug) LOG.debug("Workload Filters: " + this.workload_filter.toString()); this.workload = new Workload(this.catalog); this.workload.load(path, this.catalog_db, this.workload_filter); this.workload_path = path; if (this.workload_filter != null) this.workload_filter.reset(); } // Workload Statistics if (this.catalog_db != null) { if (this.params.containsKey(PARAM_STATS)) { this.stats = new WorkloadStatistics(this.catalog_db); String path = this.params.get(PARAM_STATS); if (debug) LOG.debug("Loading in workload statistics from '" + path + "'"); this.stats_path = new File(path); try { this.stats.load(this.stats_path, this.catalog_db); } catch (Throwable ex) { throw new RuntimeException("Failed to load stats file '" + this.stats_path + "'", ex); } } // Scaling if (this.params.containsKey(PARAM_STATS_SCALE_FACTOR) && this.stats != null) { double scale_factor = this.getDoubleParam(PARAM_STATS_SCALE_FACTOR); LOG.info("Scaling TableStatistics: " + scale_factor); AbstractTableStatisticsGenerator generator = AbstractTableStatisticsGenerator .factory(this.catalog_db, this.catalog_type, scale_factor); generator.apply(this.stats); } } } public void updateCatalog(Catalog catalog, File catalog_path) { this.catalog = catalog; this.catalogContext = new CatalogContext(catalog, catalog_path); this.catalog_db = CatalogUtil.getDatabase(catalog); if (catalog_path != null) this.catalog_path = catalog_path; } /** * @param args * @throws Exception */ @SuppressWarnings("unchecked") public void process(String[] args, String... required) throws Exception { final boolean debug = LOG.isDebugEnabled(); if (debug) LOG.debug("Processing " + args.length + " parameters..."); final Pattern p = Pattern.compile("="); for (int i = 0, cnt = args.length; i < cnt; i++) { final String arg = args[i]; final String[] parts = p.split(arg, 2); if (parts[0].startsWith("-")) parts[0] = parts[0].substring(1); if (parts.length == 1) { if (parts[0].startsWith("${") == false) this.opt_params.add(parts[0]); continue; } else if (parts[0].equalsIgnoreCase("tag")) { continue; } else if (parts[1].startsWith("${") || parts[0].startsWith("#")) { continue; } if (debug) LOG.debug(String.format("%-35s = %s", parts[0], parts[1])); // DesignerHints Override if (parts[0].startsWith(PARAM_DESIGNER_HINTS_PREFIX)) { String param = parts[0].replace(PARAM_DESIGNER_HINTS_PREFIX, "").toLowerCase(); try { Field f = DesignerHints.class.getField(param); this.hints_params.put(f.getName(), parts[1]); if (debug) LOG.debug(String.format("DesignerHints.%s = %s", param, parts[1])); } catch (NoSuchFieldException ex) { throw new Exception("Unknown DesignerHints parameter: " + param, ex); } } // HStoreConf Parameter else if (HStoreConf.isConfParameter(parts[0])) { this.conf_params.put(parts[0].toLowerCase(), parts[1]); } // ArgumentsParser Parameter else if (PARAMS.contains(parts[0].toLowerCase())) { this.params.put(parts[0].toLowerCase(), parts[1]); } // Invalid! else { String suggestions = ""; i = 0; String end = CollectionUtil.last(parts[0].split("\\.")); for (String param : PARAMS) { String param_end = CollectionUtil.last(param.split("\\.")); if (param.startsWith(parts[0]) || (end != null && param.endsWith(end)) || (end != null && param_end != null && param_end.startsWith(end))) { if (suggestions.isEmpty()) suggestions = ". Possible Matches:"; suggestions += String.format("\n [%02d] %s", ++i, param); } } // FOR throw new Exception("Unknown parameter '" + parts[0] + "'" + suggestions); } } // FOR // ------------------------------------------------------- // CATALOGS // ------------------------------------------------------- // Text File if (this.params.containsKey(PARAM_CATALOG)) { String path = this.params.get(PARAM_CATALOG); if (debug) LOG.debug("Loading catalog from file '" + path + "'"); Catalog catalog = CatalogUtil.loadCatalog(path); if (catalog == null) throw new Exception("Failed to load catalog object from file '" + path + "'"); this.updateCatalog(catalog, new File(path)); } // Jar File else if (this.params.containsKey(PARAM_CATALOG_JAR)) { String path = this.params.get(PARAM_CATALOG_JAR); this.params.put(PARAM_CATALOG, path); File jar_file = new File(path); Catalog catalog = CatalogUtil.loadCatalogFromJar(path); if (catalog == null) throw new Exception("Failed to load catalog object from jar file '" + path + "'"); if (debug) LOG.debug("Loaded catalog from jar file '" + path + "'"); this.updateCatalog(catalog, jar_file); if (!this.params.containsKey(PARAM_CATALOG_TYPE)) { String jar_name = jar_file.getName(); int jar_idx = jar_name.lastIndexOf(".jar"); if (jar_idx != -1) { ProjectType type = ProjectType.get(jar_name.substring(0, jar_idx)); if (type != null) { if (debug) LOG.debug("Set catalog type '" + type + "' from catalog jar file name"); this.catalog_type = type; this.params.put(PARAM_CATALOG_TYPE, this.catalog_type.toString()); } } } // HACK: Extract the ParameterMappings embedded in jar and write them to a temp file // This is terrible, confusing, and a total mess... // I have no one to blame but myself... JarReader reader = new JarReader(jar_file.getAbsolutePath()); for (String file : reader.getContentsFromJarfile()) { if (file.endsWith(".mappings")) { String contents = new String(JarReader.readFileFromJarAtURL(jar_file.getAbsolutePath(), file)); File copy = FileUtil.writeStringToTempFile(contents, "mappings", true); this.params.put(PARAM_MAPPINGS, copy.toString()); break; } } // FOR } // Schema File else if (this.params.containsKey(PARAM_CATALOG_SCHEMA)) { String path = this.params.get(PARAM_CATALOG_SCHEMA); Catalog catalog = CompilerUtil.compileCatalog(path); if (catalog == null) throw new Exception("Failed to load schema from '" + path + "'"); if (debug) LOG.debug("Loaded catalog from schema file '" + path + "'"); this.updateCatalog(catalog, new File(path)); } // Catalog Type if (this.params.containsKey(PARAM_CATALOG_TYPE)) { String catalog_type = this.params.get(PARAM_CATALOG_TYPE); ProjectType type = ProjectType.get(catalog_type); if (type == null) { throw new Exception("Unknown catalog type '" + catalog_type + "'"); } this.catalog_type = type; } // Update Cluster Configuration if (this.params.containsKey(ArgumentsParser.PARAM_CATALOG_HOSTS) && DISABLE_UPDATE_CATALOG == false) { ClusterConfiguration cc = new ClusterConfiguration(this.getParam(ArgumentsParser.PARAM_CATALOG_HOSTS)); this.updateCatalog(FixCatalog.cloneCatalog(this.catalog, cc), null); } // Check the requirements after loading the catalog, because some of the // above parameters will set the catalog one if (required != null && required.length > 0) this.require(required); // ------------------------------------------------------- // PHYSICAL DESIGN COMPONENTS // ------------------------------------------------------- if (this.params.containsKey(PARAM_PARTITION_PLAN)) { assert (this.catalog_db != null); File path = new File(this.params.get(PARAM_PARTITION_PLAN)); boolean ignoreMissing = this.getBooleanParam(ArgumentsParser.PARAM_PARTITION_PLAN_IGNORE_MISSING, false); if (path.exists() || (path.exists() == false && ignoreMissing == false)) { if (debug) LOG.debug("Loading in partition plan from '" + path + "'"); this.pplan = new PartitionPlan(); this.pplan.load(path, this.catalog_db); // Apply! if (this.params.containsKey(PARAM_PARTITION_PLAN_APPLY) && this.getBooleanParam(PARAM_PARTITION_PLAN_APPLY)) { boolean secondaryIndexes = this.getBooleanParam(PARAM_PARTITION_PLAN_NO_SECONDARY, false) == false; LOG.info(String.format("Applying PartitionPlan '%s' to catalog [enableSecondaryIndexes=%s]", path.getName(), secondaryIndexes)); this.pplan.apply(this.catalog_db, secondaryIndexes); } } } // ------------------------------------------------------- // DESIGNER COMPONENTS // ------------------------------------------------------- if (this.params.containsKey(PARAM_DESIGNER_THREADS)) { this.max_concurrent = Integer.valueOf(this.params.get(PARAM_DESIGNER_THREADS)); } if (this.params.containsKey(PARAM_DESIGNER_INTERVALS)) { this.num_intervals = Integer.valueOf(this.params.get(PARAM_DESIGNER_INTERVALS)); } if (this.params.containsKey(PARAM_DESIGNER_HINTS)) { File path = this.getFileParam(PARAM_DESIGNER_HINTS); if (debug) LOG.debug("Loading in designer hints from '" + path + "'.\nForced Values:\n" + StringUtil.formatMaps(this.hints_params)); this.designer_hints.load(path, catalog_db, this.hints_params); } if (this.params.containsKey(PARAM_DESIGNER_CHECKPOINT)) { this.designer_checkpoint = new File(this.params.get(PARAM_DESIGNER_CHECKPOINT)); } String designer_attributes[] = { PARAM_DESIGNER_PARTITIONER, PARAM_DESIGNER_MAPPER, PARAM_DESIGNER_INDEXER, PARAM_DESIGNER_COSTMODEL }; ClassLoader loader = ClassLoader.getSystemClassLoader(); for (String key : designer_attributes) { if (this.params.containsKey(key)) { String target_name = this.params.get(key); Class<?> target_class = loader.loadClass(target_name); assert (target_class != null); if (debug) LOG.debug("Set " + key + " class to " + target_class.getName()); if (key.equals(PARAM_DESIGNER_PARTITIONER)) { this.partitioner_class = (Class<? extends AbstractPartitioner>) target_class; } else if (key.equals(PARAM_DESIGNER_MAPPER)) { this.mapper_class = (Class<? extends AbstractMapper>) target_class; } else if (key.equals(PARAM_DESIGNER_INDEXER)) { this.indexer_class = (Class<? extends AbstractIndexSelector>) target_class; } else if (key.equals(PARAM_DESIGNER_COSTMODEL)) { this.costmodel_class = (Class<? extends AbstractCostModel>) target_class; // Special Case: TimeIntervalCostModel if (target_name.endsWith(TimeIntervalCostModel.class.getSimpleName())) { this.costmodel = new TimeIntervalCostModel<SingleSitedCostModel>(this.catalogContext, SingleSitedCostModel.class, this.num_intervals); } else { this.costmodel = ClassUtil.newInstance(this.costmodel_class, new Object[] { this.catalogContext }, new Class[] { Database.class }); } } else { assert (false) : "Invalid key '" + key + "'"; } } } // FOR // ------------------------------------------------------- // TRANSACTION ESTIMATION COMPONENTS // ------------------------------------------------------- if (this.params.containsKey(PARAM_MAPPINGS) && DISABLE_UPDATE_CATALOG == false) { assert (this.catalog_db != null); File path = new File(this.params.get(PARAM_MAPPINGS)); if (path.exists()) { this.param_mappings.load(path, this.catalog_db); } else { LOG.warn("The ParameterMappings file '" + path + "' does not exist"); } } if (this.params.containsKey(PARAM_MARKOV_THRESHOLDS_VALUE)) { assert (this.catalog_db != null); float defaultValue = this.getDoubleParam(PARAM_MARKOV_THRESHOLDS_VALUE).floatValue(); this.thresholds = new EstimationThresholds(defaultValue); this.params.put(PARAM_MARKOV_THRESHOLDS, this.thresholds.toString()); LOG.debug("CREATED THRESHOLDS: " + this.thresholds); } else if (this.params.containsKey(PARAM_MARKOV_THRESHOLDS)) { assert (this.catalog_db != null); this.thresholds = new EstimationThresholds(); File path = new File(this.params.get(PARAM_MARKOV_THRESHOLDS)); if (path.exists()) { this.thresholds.load(path, this.catalog_db); } else { LOG.warn("The estimation thresholds file '" + path + "' does not exist"); } LOG.debug("LOADED THRESHOLDS: " + this.thresholds); } // ------------------------------------------------------- // HASHER // ------------------------------------------------------- if (this.catalog != null) { if (this.params.containsKey(PARAM_HASHER_CLASS)) { String hasherClassName = this.params.get(PARAM_HASHER_CLASS); this.hasher_class = (Class<? extends AbstractHasher>) loader.loadClass(hasherClassName); } Class<?> paramClasses[] = new Class[] { CatalogContext.class, int.class }; Object paramValues[] = new Object[] { this.catalogContext, this.catalogContext.numberOfPartitions }; Constructor<? extends AbstractHasher> constructor = this.hasher_class.getConstructor(paramClasses); this.hasher = constructor.newInstance(paramValues); if (!(this.hasher instanceof DefaultHasher)) LOG.debug("Loaded hasher " + this.hasher.getClass()); if (this.params.containsKey(PARAM_HASHER_PROFILE)) { this.hasher.load(this.getFileParam(PARAM_HASHER_PROFILE), null); } } // ------------------------------------------------------- // SAMPLE WORKLOAD TRACE // ------------------------------------------------------- this.loadWorkload(); } @Override public String toString() { Map<String, Object> m0 = new ListOrderedMap<String, Object>(); m0.putAll(this.params); Map<String, Object> m1 = null; int opt_cnt = this.opt_params.size(); if (opt_cnt > 0) { Map<String, Object> opt_inner = new ListOrderedMap<String, Object>(); for (int i = 0; i < opt_cnt; i++) { opt_inner.put(String.format("#%02d", i), this.opt_params.get(i)); } m1 = new ListOrderedMap<String, Object>(); m1.put(String.format("Optional Parameters [%d]", opt_cnt), StringUtil.formatMaps(opt_inner)); } return StringUtil.formatMaps(m0, m1); } }