eu.socialsensor.main.BenchmarkConfiguration.java Source code

Java tutorial

Introduction

Here is the source code for eu.socialsensor.main.BenchmarkConfiguration.java

Source

package eu.socialsensor.main;

import java.io.File;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;

import org.apache.commons.configuration.Configuration;
import org.apache.commons.math3.util.CombinatoricsUtils;

import com.amazon.titan.diskstorage.dynamodb.BackendDataModel;
import com.amazon.titan.diskstorage.dynamodb.Constants;
import com.google.common.primitives.Ints;
import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration;

import eu.socialsensor.dataset.DatasetFactory;

/**
 * 
 * @author Alexander Patrikalakis
 *
 */
public class BenchmarkConfiguration {
    // orientdb Configuration
    private static final String LIGHTWEIGHT_EDGES = "lightweight-edges";

    // Sparksee / DEX configuration
    private static final String LICENSE_KEY = "license-key";

    // Titan specific configuration
    private static final String TITAN = "titan";
    private static final String BUFFER_SIZE = GraphDatabaseConfiguration.BUFFER_SIZE.getName();
    private static final String IDS_BLOCKSIZE = GraphDatabaseConfiguration.IDS_BLOCK_SIZE.getName();
    private static final String PAGE_SIZE = GraphDatabaseConfiguration.PAGE_SIZE.getName();
    public static final String CSV_INTERVAL = GraphDatabaseConfiguration.METRICS_CSV_INTERVAL.getName();
    public static final String CSV = GraphDatabaseConfiguration.METRICS_CSV_NS.getName();
    private static final String CSV_DIR = GraphDatabaseConfiguration.METRICS_CSV_DIR.getName();
    public static final String GRAPHITE = GraphDatabaseConfiguration.METRICS_GRAPHITE_NS.getName();
    private static final String GRAPHITE_HOSTNAME = GraphDatabaseConfiguration.GRAPHITE_HOST.getName();

    // DynamoDB Storage Backend for Titan specific configuration
    private static final String CONSTRUCTOR_ARGS = Constants.DYNAMODB_CREDENTIALS_CONSTRUCTOR_ARGS.getName();
    private static final String CLASS_NAME = Constants.DYNAMODB_CREDENTIALS_CLASS_NAME.getName();
    private static final String CONSISTENT_READ = Constants.DYNAMODB_FORCE_CONSISTENT_READ.getName();
    private static final String TPS = "tps";
    private static final String CREDENTIALS = Constants.DYNAMODB_CLIENT_CREDENTIALS_NAMESPACE.getName();
    private static final String ENDPOINT = Constants.DYNAMODB_CLIENT_ENDPOINT.getName();
    private static final String TABLE_PREFIX = Constants.DYNAMODB_TABLE_PREFIX.getName();

    // benchmark configuration
    private static final String DATASET = "dataset";
    private static final String DATABASE_STORAGE_DIRECTORY = "database-storage-directory";
    private static final String ACTUAL_COMMUNITIES = "actual-communities";
    private static final String NODES_COUNT = "nodes-count";
    private static final String RANDOMIZE_CLUSTERING = "randomize-clustering";
    private static final String CACHE_VALUES = "cache-values";
    private static final String CACHE_INCREMENT_FACTOR = "cache-increment-factor";
    private static final String CACHE_VALUES_COUNT = "cache-values-count";
    private static final String PERMUTE_BENCHMARKS = "permute-benchmarks";
    private static final String RANDOM_NODES = "shortest-path-random-nodes";

    private static final Set<String> metricsReporters = new HashSet<String>();
    static {
        metricsReporters.add(CSV);
        metricsReporters.add(GRAPHITE);
    }

    private final File dataset;
    private final List<BenchmarkType> benchmarkTypes;
    private final SortedSet<GraphDatabaseType> selectedDatabases;
    private final File resultsPath;

    // storage directory
    private final File dbStorageDirectory;

    // metrics (optional)
    private final long csvReportingInterval;
    private final File csvDir;
    private final String graphiteHostname;
    private final long graphiteReportingInterval;

    // storage backend specific settings
    private final long dynamodbTps;
    private final BackendDataModel dynamodbDataModel;
    private final boolean dynamodbConsistentRead;
    private final Boolean orientLightweightEdges;
    private final String sparkseeLicenseKey;

    // shortest path
    private final int randomNodes;

    // clustering
    private final Boolean randomizedClustering;
    private final Integer nodesCount;
    private final Integer cacheValuesCount;
    private final Double cacheIncrementFactor;
    private final List<Integer> cacheValues;
    private final File actualCommunities;
    private final boolean permuteBenchmarks;
    private final int scenarios;
    private final String dynamodbCredentialsFqClassName;
    private final String dynamodbCredentialsCtorArguments;
    private final String dynamodbEndpoint;
    private final int bufferSize;
    private final int blocksize;
    private final int pageSize;
    private final int dynamodbWorkerThreads;
    private final boolean dynamodbPrecreateTables;
    private final String dynamodbTablePrefix;

    public String getDynamodbCredentialsFqClassName() {
        return dynamodbCredentialsFqClassName;
    }

    public String getDynamodbCredentialsCtorArguments() {
        return dynamodbCredentialsCtorArguments;
    }

    public String getDynamodbEndpoint() {
        return dynamodbEndpoint;
    }

    public BenchmarkConfiguration(Configuration appconfig) {
        if (appconfig == null) {
            throw new IllegalArgumentException("appconfig may not be null");
        }

        Configuration eu = appconfig.subset("eu");
        Configuration socialsensor = eu.subset("socialsensor");

        //metrics
        final Configuration metrics = socialsensor.subset(GraphDatabaseConfiguration.METRICS_NS.getName());

        final Configuration graphite = metrics.subset(GRAPHITE);
        this.graphiteHostname = graphite.getString(GRAPHITE_HOSTNAME, null);
        this.graphiteReportingInterval = graphite.getLong(GraphDatabaseConfiguration.GRAPHITE_INTERVAL.getName(),
                1000 /*default 1sec*/);

        final Configuration csv = metrics.subset(CSV);
        this.csvReportingInterval = metrics.getLong(CSV_INTERVAL, 1000 /*ms*/);
        this.csvDir = csv.containsKey(CSV_DIR)
                ? new File(csv.getString(CSV_DIR, System.getProperty("user.dir") /*default*/))
                : null;

        Configuration dynamodb = socialsensor.subset("dynamodb");
        this.dynamodbWorkerThreads = dynamodb.getInt("workers", 25);
        Configuration credentials = dynamodb.subset(CREDENTIALS);
        this.dynamodbPrecreateTables = dynamodb.getBoolean("precreate-tables", Boolean.FALSE);
        this.dynamodbTps = Math.max(1, dynamodb.getLong(TPS, 750 /*default*/));
        this.dynamodbConsistentRead = dynamodb.containsKey(CONSISTENT_READ) ? dynamodb.getBoolean(CONSISTENT_READ)
                : false;
        this.dynamodbDataModel = dynamodb.containsKey("data-model")
                ? BackendDataModel.valueOf(dynamodb.getString("data-model"))
                : null;
        this.dynamodbCredentialsFqClassName = credentials.containsKey(CLASS_NAME)
                ? credentials.getString(CLASS_NAME)
                : null;
        this.dynamodbCredentialsCtorArguments = credentials.containsKey(CONSTRUCTOR_ARGS)
                ? credentials.getString(CONSTRUCTOR_ARGS)
                : null;
        this.dynamodbEndpoint = dynamodb.containsKey(ENDPOINT) ? dynamodb.getString(ENDPOINT) : null;
        this.dynamodbTablePrefix = dynamodb.containsKey(TABLE_PREFIX) ? dynamodb.getString(TABLE_PREFIX)
                : Constants.DYNAMODB_TABLE_PREFIX.getDefaultValue();

        Configuration orient = socialsensor.subset("orient");
        orientLightweightEdges = orient.containsKey(LIGHTWEIGHT_EDGES) ? orient.getBoolean(LIGHTWEIGHT_EDGES)
                : null;

        Configuration sparksee = socialsensor.subset("sparksee");
        sparkseeLicenseKey = sparksee.containsKey(LICENSE_KEY) ? sparksee.getString(LICENSE_KEY) : null;

        Configuration titan = socialsensor.subset(TITAN); //TODO(amcp) move dynamodb ns into titan
        bufferSize = titan.getInt(BUFFER_SIZE, GraphDatabaseConfiguration.BUFFER_SIZE.getDefaultValue());
        blocksize = titan.getInt(IDS_BLOCKSIZE, GraphDatabaseConfiguration.IDS_BLOCK_SIZE.getDefaultValue());
        pageSize = titan.getInt(PAGE_SIZE, GraphDatabaseConfiguration.PAGE_SIZE.getDefaultValue());

        // database storage directory
        if (!socialsensor.containsKey(DATABASE_STORAGE_DIRECTORY)) {
            throw new IllegalArgumentException("configuration must specify database-storage-directory");
        }
        dbStorageDirectory = new File(socialsensor.getString(DATABASE_STORAGE_DIRECTORY));
        dataset = validateReadableFile(socialsensor.getString(DATASET), DATASET);

        // load the dataset
        DatasetFactory.getInstance().getDataset(dataset);

        if (!socialsensor.containsKey(PERMUTE_BENCHMARKS)) {
            throw new IllegalArgumentException("configuration must set permute-benchmarks to true or false");
        }
        permuteBenchmarks = socialsensor.getBoolean(PERMUTE_BENCHMARKS);

        List<?> benchmarkList = socialsensor.getList("benchmarks");
        benchmarkTypes = new ArrayList<BenchmarkType>();
        for (Object str : benchmarkList) {
            benchmarkTypes.add(BenchmarkType.valueOf(str.toString()));
        }

        selectedDatabases = new TreeSet<GraphDatabaseType>();
        for (Object database : socialsensor.getList("databases")) {
            if (!GraphDatabaseType.STRING_REP_MAP.keySet().contains(database.toString())) {
                throw new IllegalArgumentException(
                        String.format("selected database %s not supported", database.toString()));
            }
            selectedDatabases.add(GraphDatabaseType.STRING_REP_MAP.get(database));
        }
        scenarios = permuteBenchmarks ? Ints.checkedCast(CombinatoricsUtils.factorial(selectedDatabases.size()))
                : 1;

        resultsPath = new File(System.getProperty("user.dir"), socialsensor.getString("results-path"));
        if (!resultsPath.exists() && !resultsPath.mkdirs()) {
            throw new IllegalArgumentException("unable to create results directory");
        }
        if (!resultsPath.canWrite()) {
            throw new IllegalArgumentException("unable to write to results directory");
        }

        randomNodes = socialsensor.getInteger(RANDOM_NODES, new Integer(100));

        if (this.benchmarkTypes.contains(BenchmarkType.CLUSTERING)) {
            if (!socialsensor.containsKey(NODES_COUNT)) {
                throw new IllegalArgumentException("the CW benchmark requires nodes-count integer in config");
            }
            nodesCount = socialsensor.getInt(NODES_COUNT);

            if (!socialsensor.containsKey(RANDOMIZE_CLUSTERING)) {
                throw new IllegalArgumentException("the CW benchmark requires randomize-clustering bool in config");
            }
            randomizedClustering = socialsensor.getBoolean(RANDOMIZE_CLUSTERING);

            if (!socialsensor.containsKey(ACTUAL_COMMUNITIES)) {
                throw new IllegalArgumentException("the CW benchmark requires a file with actual communities");
            }
            actualCommunities = validateReadableFile(socialsensor.getString(ACTUAL_COMMUNITIES),
                    ACTUAL_COMMUNITIES);

            final boolean notGenerating = socialsensor.containsKey(CACHE_VALUES);
            if (notGenerating) {
                List<?> objects = socialsensor.getList(CACHE_VALUES);
                cacheValues = new ArrayList<Integer>(objects.size());
                cacheValuesCount = null;
                cacheIncrementFactor = null;
                for (Object o : objects) {
                    cacheValues.add(Integer.valueOf(o.toString()));
                }
            } else if (socialsensor.containsKey(CACHE_VALUES_COUNT)
                    && socialsensor.containsKey(CACHE_INCREMENT_FACTOR)) {
                cacheValues = null;
                // generate the cache values with parameters
                if (!socialsensor.containsKey(CACHE_VALUES_COUNT)) {
                    throw new IllegalArgumentException(
                            "the CW benchmark requires cache-values-count int in config when cache-values not specified");
                }
                cacheValuesCount = socialsensor.getInt(CACHE_VALUES_COUNT);

                if (!socialsensor.containsKey(CACHE_INCREMENT_FACTOR)) {
                    throw new IllegalArgumentException(
                            "the CW benchmark requires cache-increment-factor int in config when cache-values not specified");
                }
                cacheIncrementFactor = socialsensor.getDouble(CACHE_INCREMENT_FACTOR);
            } else {
                throw new IllegalArgumentException(
                        "when doing CW benchmark, must provide cache-values or parameters to generate them");
            }
        } else {
            randomizedClustering = null;
            nodesCount = null;
            cacheValuesCount = null;
            cacheIncrementFactor = null;
            cacheValues = null;
            actualCommunities = null;
        }
    }

    public File getDataset() {
        return dataset;
    }

    public SortedSet<GraphDatabaseType> getSelectedDatabases() {
        return selectedDatabases;
    }

    public File getDbStorageDirectory() {
        return dbStorageDirectory;
    }

    public File getResultsPath() {
        return resultsPath;
    }

    public long getDynamodbTps() {
        return dynamodbTps;
    }

    public boolean dynamodbConsistentRead() {
        return dynamodbConsistentRead;
    }

    public BackendDataModel getDynamodbDataModel() {
        return dynamodbDataModel;
    }

    public List<BenchmarkType> getBenchmarkTypes() {
        return benchmarkTypes;
    }

    public Boolean randomizedClustering() {
        return randomizedClustering;
    }

    public Integer getNodesCount() {
        return nodesCount;
    }

    public Integer getCacheValuesCount() {
        return cacheValuesCount;
    }

    public Double getCacheIncrementFactor() {
        return cacheIncrementFactor;
    }

    public List<Integer> getCacheValues() {
        return cacheValues;
    }

    public File getActualCommunitiesFile() {
        return actualCommunities;
    }

    public Boolean orientLightweightEdges() {
        return orientLightweightEdges;
    }

    public String getSparkseeLicenseKey() {
        return sparkseeLicenseKey;
    }

    public boolean permuteBenchmarks() {
        return permuteBenchmarks;
    }

    public int getScenarios() {
        return scenarios;
    }

    private static final File validateReadableFile(String fileName, String fileType) {
        File file = new File(fileName);
        if (!file.exists()) {
            throw new IllegalArgumentException(String.format("the %s does not exist", fileType));
        }

        if (!(file.isFile() && file.canRead())) {
            throw new IllegalArgumentException(
                    String.format("the %s must be a file that this user can read", fileType));
        }
        return file;
    }

    public int getRandomNodes() {
        return randomNodes;
    }

    public long getCsvReportingInterval() {
        return csvReportingInterval;
    }

    public long getGraphiteReportingInterval() {
        return graphiteReportingInterval;
    }

    public File getCsvDir() {
        return csvDir;
    }

    public String getGraphiteHostname() {
        return graphiteHostname;
    }

    public int getTitanBufferSize() {
        return bufferSize;
    }

    public int getTitanIdsBlocksize() {
        return blocksize;
    }

    public int getTitanPageSize() {
        return pageSize;
    }

    public int getDynamodbWorkerThreads() {
        return dynamodbWorkerThreads;
    }

    public boolean getDynamodbPrecreateTables() {
        return dynamodbPrecreateTables;
    }

    public String getDynamodbTablePrefix() {
        return dynamodbTablePrefix;
    }

    public boolean publishCsvMetrics() {
        return csvDir != null;
    }

    public boolean publishGraphiteMetrics() {
        return graphiteHostname != null && !graphiteHostname.isEmpty();
    }
}