Example usage for org.apache.hadoop.conf Configuration get

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration get.

Prototype

public String get(String name) 

Source Link

Document

Get the value of the name property, null if no such property exists.

Usage

From source file:andromache.config.CassandraConfigHelper.java

License:Apache License

public static void setDefaultReadConsistencyLevel(Configuration configuration) {
    if (configuration.get(CASSANDRA_CONSISTENCYLEVEL_READ) == null) {
        configuration.set(CASSANDRA_CONSISTENCYLEVEL_READ, DEFAULT_CONSISTENCY_LEVEL);
    }//from  w ww  . ja  v a  2 s . c  o m
}

---

From source file:andromache.config.CassandraConfigHelper.java

License:Apache License

public static String getOutputKeyspaceUserName(Configuration configuration, String keyspace) {
    return configuration.get(OUTPUT_KEYSPACE_USER_NAME_KEY + ":" + keyspace);
}

---

From source file:andromache.config.CassandraConfigHelper.java

License:Apache License

public static String getOutputKeyspacePassword(Configuration configuration, String keyspace) {
    return configuration.get(OUTPUT_KEYSPACE_USER_PASSWORD_KEY + ":" + keyspace);
}

---

From source file:apex.benchmark.ApplicationDimensionComputation.java

License:Apache License

protected AppDataSingleSchemaDimensionStoreHDHT createStore(DAG dag, Configuration conf, String eventSchema) {
    AppDataSingleSchemaDimensionStoreHDHT store = dag.addOperator("Store", ProcessTimeAwareStore.class);
    store.setUpdateEnumValues(true);/*w w w . j  ava 2  s.co  m*/
    String basePath = Preconditions.checkNotNull(conf.get(PROP_STORE_PATH),
            "base path should be specified in the properties.xml");
    TFileImpl hdsFile = new TFileImpl.DTFileImpl();
    basePath += System.currentTimeMillis();
    hdsFile.setBasePath(basePath);

    store.setFileStore(hdsFile);
    dag.setAttribute(store, Context.OperatorContext.COUNTERS_AGGREGATOR,
            new BasicCounters.LongAggregator<MutableLong>());
    store.setConfigurationSchemaJSON(eventSchema);
    store.setPartitionCount(storePartitionCount);
    if (storePartitionCount > 1) {
        store.setPartitionCount(storePartitionCount);
        store.setQueryResultUnifier(new DimensionStoreHDHTNonEmptyQueryResultUnifier());
    }
    return store;
}

---

From source file:apex.benchmark.ApplicationWithDCWithoutDeserializer.java

License:Apache License

@Override
public void populateDAG(DAG dag, Configuration configuration) {
    redisServer = configuration.get("dt.application.AppWithDCWithoutDe.redisServer");

    DefaultOutputPort<DimensionTuple> upstreamOutput = populateUpstreamDAG(dag, configuration);

    //populateHardCodedDimensionsDAG(dag, configuration, generateOperator.outputPort);
    populateDimensionsDAG(dag, configuration, upstreamOutput);
}

---

From source file:apex.benchmark.ApplicationWithGenerator.java

License:Apache License

@Override
public void populateDAG(DAG dag, Configuration configuration) {
    // Create operators for each step
    // settings are applied by the platform using the config file.
    JsonGenerator eventGenerator = dag.addOperator("eventGenerator", new JsonGenerator());
    FilterTuples filterTuples = dag.addOperator("filterTuples", new FilterTuples());
    FilterFields filterFields = dag.addOperator("filterFields", new FilterFields());
    RedisJoin redisJoin = dag.addOperator("redisJoin", new RedisJoin());
    CampaignProcessor campaignProcessor = dag.addOperator("campaignProcessor", new CampaignProcessor());

    eventGenerator.setNumAdsPerCampaign(Integer.parseInt(configuration.get("numberOfAds")));
    eventGenerator.setNumCampaigns(Integer.parseInt(configuration.get("numberOfCampaigns")));
    setupRedis(eventGenerator.getCampaigns(), configuration.get("redis"));

    // Connect the Ports in the Operators
    dag.addStream("filterTuples", eventGenerator.out, filterTuples.input)
            .setLocality(DAG.Locality.CONTAINER_LOCAL);
    dag.addStream("filterFields", filterTuples.output, filterFields.input)
            .setLocality(DAG.Locality.CONTAINER_LOCAL);
    dag.addStream("redisJoin", filterFields.output, redisJoin.input).setLocality(DAG.Locality.CONTAINER_LOCAL);
    dag.addStream("output", redisJoin.output, campaignProcessor.input);

    dag.setInputPortAttribute(filterTuples.input, Context.PortContext.PARTITION_PARALLEL, true);
    dag.setInputPortAttribute(filterFields.input, Context.PortContext.PARTITION_PARALLEL, true);
    dag.setInputPortAttribute(redisJoin.input, Context.PortContext.PARTITION_PARALLEL, true);

    dag.setAttribute(eventGenerator, Context.OperatorContext.PARTITIONER,
            new StatelessPartitioner<EventGenerator>(8));
    dag.setAttribute(campaignProcessor, Context.OperatorContext.PARTITIONER,
            new StatelessPartitioner<CampaignProcessor>(8));
}

---

From source file:apex.benchmark.ConfigUtil.java

License:Apache License

public static String getGatewayAddress(DAG dag, Configuration conf) {
    String gatewayAddress = dag.getValue(DAGContext.GATEWAY_CONNECT_ADDRESS);
    if (gatewayAddress == null) {
        gatewayAddress = conf.get(PROP_GATEWAY_ADDRESS);
    }/*  w ww.  j av a  2 s .c o m*/
    return gatewayAddress;
}

---

From source file:at.illecker.hama.hybrid.examples.hellohybrid.HelloHybridBSP.java

License:Apache License

public static void main(String[] args) throws InterruptedException, IOException, ClassNotFoundException {

    Configuration conf = new HamaConfiguration();

    if (args.length > 0) {
        if (args.length == 1) {
            conf.setInt("bsp.peers.num", Integer.parseInt(args[0]));
        } else {//from w ww.jav  a2s  . co  m
            System.out.println("Wrong argument size!");
            System.out.println("    Argument1=numBspTask");
            return;
        }
    } else {
        // BSPJobClient jobClient = new BSPJobClient(conf);
        // ClusterStatus cluster = jobClient.getClusterStatus(true);
        // job.setNumBspTask(cluster.getMaxTasks());

        conf.setInt("bsp.peers.num", 2); // 1 CPU and 1 GPU
    }
    // Enable one GPU task
    conf.setInt("bsp.peers.gpu.num", 1);
    conf.setBoolean("hama.pipes.logging", true);

    LOG.info("NumBspTask: " + conf.getInt("bsp.peers.num", 0));
    LOG.info("NumBspGpuTask: " + conf.getInt("bsp.peers.gpu.num", 0));
    LOG.info("bsp.tasks.maximum: " + conf.get("bsp.tasks.maximum"));
    LOG.info("inputPath: " + CONF_INPUT_DIR);
    LOG.info("outputPath: " + CONF_OUTPUT_DIR);

    Path example = new Path(CONF_INPUT_DIR.getParent(), "example.seq");
    conf.set(CONF_EXAMPLE_PATH, example.toString());
    LOG.info("exampleFile: " + example.toString());

    prepareInput(conf, CONF_INPUT_DIR, example, CONF_N);

    BSPJob job = createHelloHybridBSPConf(conf, CONF_INPUT_DIR, CONF_OUTPUT_DIR);

    long startTime = System.currentTimeMillis();
    if (job.waitForCompletion(true)) {
        LOG.info("Job Finished in " + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds");

        // Print input files
        // printOutput(job, CONF_INPUT_DIR);
        // printOutput(job, example);

        // Print output
        printOutput(job, FileOutputFormat.getOutputPath(job));
    }
}

---

From source file:at.illecker.hama.hybrid.examples.kmeans.KMeansHybridBSP.java

License:Apache License

public static void main(String[] args) throws Exception {

    // Defaults/*from   w w  w.  ja  v a2s.  co  m*/
    int numBspTask = 1;
    int numGpuBspTask = 1;
    int blockSize = BLOCK_SIZE;
    int gridSize = GRID_SIZE;
    long n = 10; // input vectors
    int k = 3; // start vectors
    int vectorDimension = 2;
    int maxIteration = 10;
    boolean useTestExampleInput = false;
    boolean isDebugging = false;
    boolean timeMeasurement = false;
    int GPUPercentage = 80;

    Configuration conf = new HamaConfiguration();
    FileSystem fs = FileSystem.get(conf);

    // Set numBspTask to maxTasks
    // BSPJobClient jobClient = new BSPJobClient(conf);
    // ClusterStatus cluster = jobClient.getClusterStatus(true);
    // numBspTask = cluster.getMaxTasks();

    if (args.length > 0) {
        if (args.length == 12) {
            numBspTask = Integer.parseInt(args[0]);
            numGpuBspTask = Integer.parseInt(args[1]);
            blockSize = Integer.parseInt(args[2]);
            gridSize = Integer.parseInt(args[3]);
            n = Long.parseLong(args[4]);
            k = Integer.parseInt(args[5]);
            vectorDimension = Integer.parseInt(args[6]);
            maxIteration = Integer.parseInt(args[7]);
            useTestExampleInput = Boolean.parseBoolean(args[8]);
            GPUPercentage = Integer.parseInt(args[9]);
            isDebugging = Boolean.parseBoolean(args[10]);
            timeMeasurement = Boolean.parseBoolean(args[11]);

        } else {
            System.out.println("Wrong argument size!");
            System.out.println("    Argument1=numBspTask");
            System.out.println("    Argument2=numGpuBspTask");
            System.out.println("    Argument3=blockSize");
            System.out.println("    Argument4=gridSize");
            System.out.println("    Argument5=n | Number of input vectors (" + n + ")");
            System.out.println("    Argument6=k | Number of start vectors (" + k + ")");
            System.out.println(
                    "    Argument7=vectorDimension | Dimension of each vector (" + vectorDimension + ")");
            System.out.println(
                    "    Argument8=maxIterations | Number of maximal iterations (" + maxIteration + ")");
            System.out.println("    Argument9=testExample | Use testExample input (true|false=default)");
            System.out.println("    Argument10=GPUPercentage (percentage of input)");
            System.out.println("    Argument11=isDebugging (true|false=defaul)");
            System.out.println("    Argument12=timeMeasurement (true|false=defaul)");
            return;
        }
    }

    // Set config variables
    conf.setBoolean(CONF_DEBUG, isDebugging);
    conf.setBoolean("hama.pipes.logging", false);
    conf.setBoolean(CONF_TIME, timeMeasurement);

    // Set CPU tasks
    conf.setInt("bsp.peers.num", numBspTask);
    // Set GPU tasks
    conf.setInt("bsp.peers.gpu.num", numGpuBspTask);
    // Set GPU blockSize and gridSize
    conf.set(CONF_BLOCKSIZE, "" + blockSize);
    conf.set(CONF_GRIDSIZE, "" + gridSize);
    // Set maxIterations for KMeans
    conf.setInt(CONF_MAX_ITERATIONS, maxIteration);
    // Set n for KMeans
    conf.setLong(CONF_N, n);
    // Set GPU workload
    conf.setInt(CONF_GPU_PERCENTAGE, GPUPercentage);

    LOG.info("NumBspTask: " + conf.getInt("bsp.peers.num", 0));
    LOG.info("NumGpuBspTask: " + conf.getInt("bsp.peers.gpu.num", 0));
    LOG.info("bsp.tasks.maximum: " + conf.get("bsp.tasks.maximum"));
    LOG.info("GPUPercentage: " + conf.get(CONF_GPU_PERCENTAGE));
    LOG.info("BlockSize: " + conf.get(CONF_BLOCKSIZE));
    LOG.info("GridSize: " + conf.get(CONF_GRIDSIZE));
    LOG.info("isDebugging: " + conf.get(CONF_DEBUG));
    LOG.info("timeMeasurement: " + conf.get(CONF_TIME));
    LOG.info("useTestExampleInput: " + useTestExampleInput);
    LOG.info("inputPath: " + CONF_INPUT_DIR);
    LOG.info("centersPath: " + CONF_CENTER_DIR);
    LOG.info("outputPath: " + CONF_OUTPUT_DIR);
    LOG.info("n: " + n);
    LOG.info("k: " + k);
    LOG.info("vectorDimension: " + vectorDimension);
    LOG.info("maxIteration: " + maxIteration);

    Path centerIn = new Path(CONF_CENTER_DIR, "center_in.seq");
    Path centerOut = new Path(CONF_CENTER_DIR, "center_out.seq");
    conf.set(CONF_CENTER_IN_PATH, centerIn.toString());
    conf.set(CONF_CENTER_OUT_PATH, centerOut.toString());

    // prepare Input
    if (useTestExampleInput) {
        // prepareTestInput(conf, fs, input, centerIn);
        prepareInputData(conf, fs, CONF_INPUT_DIR, centerIn, numBspTask, numGpuBspTask, n, k, vectorDimension,
                null, GPUPercentage);
    } else {
        prepareInputData(conf, fs, CONF_INPUT_DIR, centerIn, numBspTask, numGpuBspTask, n, k, vectorDimension,
                new Random(3337L), GPUPercentage);
    }

    BSPJob job = createKMeansHybridBSPConf(conf, CONF_INPUT_DIR, CONF_OUTPUT_DIR);

    long startTime = System.currentTimeMillis();
    if (job.waitForCompletion(true)) {
        LOG.info("Job Finished in " + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds");

        if (isDebugging) {
            printFile(conf, fs, centerOut, new PipesVectorWritable(), NullWritable.get());
            printOutput(conf, fs, ".log", new IntWritable(), new PipesVectorWritable());
        }

        if (k < 50) {
            printFile(conf, fs, centerOut, new PipesVectorWritable(), NullWritable.get());
        }
    }
}

---

From source file:at.illecker.hama.hybrid.examples.matrixmultiplication2.MatrixMultiplicationHybridBSP.java

License:Apache License

public static void main(String[] args) throws Exception {

    // Defaults/* ww w.ja  v  a  2 s  . com*/
    int numBspTask = 1;
    int numGpuBspTask = 1;
    int numRowsA = 4;// 1024;
    int numColsA = 4;// 1024;
    int numRowsB = 4;// 1024;
    int numColsB = 4;// 1024;
    int tileWidth = 32; // 2 * 32 = 1024 threads matches the blocksize
    int GPUPercentage = 100;
    boolean isDebugging = true;

    Configuration conf = new HamaConfiguration();

    if (args.length > 0) {
        if (args.length == 9) {
            numBspTask = Integer.parseInt(args[0]);
            numGpuBspTask = Integer.parseInt(args[1]);
            numRowsA = Integer.parseInt(args[2]);
            numColsA = Integer.parseInt(args[3]);
            numRowsB = Integer.parseInt(args[4]);
            numColsB = Integer.parseInt(args[5]);
            tileWidth = Integer.parseInt(args[6]);
            GPUPercentage = Integer.parseInt(args[7]);
            isDebugging = Boolean.parseBoolean(args[8]);

        } else {
            System.out.println("Wrong argument size!");
            System.out.println("    Argument1=numBspTask");
            System.out.println("    Argument2=numGpuBspTask");
            System.out.println("    Argument3=numRowsA | Number of rows of the first input matrix");
            System.out.println("    Argument4=numColsA | Number of columns of the first input matrix");
            System.out.println("    Argument5=numRowsB | Number of rows of the second input matrix");
            System.out.println("    Argument6=numColsB | Number of columns of the second input matrix");
            System.out.println("    Argument7=tileWidth | TileWidth denotes the size of a submatrix");
            System.out.println("    Argument8=GPUPercentage (percentage of input)");
            System.out.println("    Argument9=debug | Enable debugging (true|false)");
            return;
        }
    }

    // Set config variables
    conf.setBoolean("hama.pipes.logging", false);
    // Set CPU tasks
    conf.setInt("bsp.peers.num", numBspTask);
    // Set GPU tasks
    conf.setInt("bsp.peers.gpu.num", numGpuBspTask);
    // Set GPU workload
    // conf.setInt(CONF_GPU_PERCENTAGE, GPUPercentage);

    LOG.info("NumBspTask: " + conf.getInt("bsp.peers.num", 0));
    LOG.info("NumGpuBspTask: " + conf.getInt("bsp.peers.gpu.num", 0));
    LOG.info("bsp.tasks.maximum: " + conf.get("bsp.tasks.maximum"));
    // LOG.info("GPUPercentage: " + conf.get(CONF_GPU_PERCENTAGE));
    LOG.info("numRowsA: " + numRowsA);
    LOG.info("numColsA: " + numColsA);
    LOG.info("numRowsB: " + numRowsB);
    LOG.info("numColsB: " + numColsB);
    LOG.info("isDebugging: " + isDebugging);
    LOG.info("inputPath: " + CONF_INPUT_DIR);
    LOG.info("outputPath: " + CONF_OUTPUT_DIR);

    if (numColsA != numRowsB) {
        throw new Exception("Cols of MatrixA != rows of MatrixB! (" + numColsA + "!=" + numRowsB + ")");
    }

    // Create random DistributedRowMatrix
    // use constant seeds to get reproducible results
    // Matrix A
    DistributedRowMatrix.createRandomDistributedRowMatrix(conf, numRowsA, numColsA, new Random(42L),
            MATRIX_A_SPLITS_PATH, false, numBspTask, numGpuBspTask, GPUPercentage);

    // Matrix B is stored in transposed order
    List<Path> transposedMatrixBPaths = DistributedRowMatrix.createRandomDistributedRowMatrix(conf, numRowsB,
            numColsB, new Random(1337L), MATRIX_B_TRANSPOSED_PATH, true);

    // Execute MatrixMultiplication BSP Job
    long startTime = System.currentTimeMillis();

    BSPJob job = MatrixMultiplicationHybridBSP.createMatrixMultiplicationHybridBSPConf(conf,
            MATRIX_A_SPLITS_PATH, transposedMatrixBPaths.get(0), MATRIX_C_PATH, tileWidth, isDebugging);

    // Multiply Matrix
    DistributedRowMatrix matrixC = null;
    if (job.waitForCompletion(true)) {

        // Rename result file to output path
        Path matrixCOutPath = new Path(MATRIX_C_PATH + "/part0.seq");

        FileSystem fs = MATRIX_C_PATH.getFileSystem(conf);
        FileStatus[] files = fs.listStatus(MATRIX_C_PATH);
        for (int i = 0; i < files.length; i++) {
            if ((files[i].getPath().getName().startsWith("part-")) && (files[i].getLen() > 97)) {
                fs.rename(files[i].getPath(), matrixCOutPath);
                break;
            }
        }

        // Read resulting Matrix from HDFS
        matrixC = new DistributedRowMatrix(matrixCOutPath, MATRIX_C_PATH, numRowsA, numColsB);
        matrixC.setConf(conf);
    }

    LOG.info("MatrixMultiplicationHybrid using Hama finished in "
            + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds");

    // Create matrix A in one file for verification
    List<Path> matrixAPaths = DistributedRowMatrix.createRandomDistributedRowMatrix(conf, numRowsA, numColsA,
            new Random(42L), MATRIX_A_PATH, false);
    DistributedRowMatrix matrixA = new DistributedRowMatrix(matrixAPaths.get(0), CONF_INPUT_DIR, numRowsA,
            numColsA);
    matrixA.setConf(conf);

    // Create matrix B, NOT transposed for verification
    List<Path> matrixBPaths = DistributedRowMatrix.createRandomDistributedRowMatrix(conf, numRowsB, numColsB,
            new Random(1337L), MATRIX_B_PATH, false);
    DistributedRowMatrix matrixB = new DistributedRowMatrix(matrixBPaths.get(0), CONF_INPUT_DIR, numRowsB,
            numColsB);
    matrixB.setConf(conf);

    // Verification
    DistributedRowMatrix matrixD = matrixA.multiplyJava(matrixB, MATRIX_D_PATH);
    if (matrixC.verify(matrixD)) {
        System.out.println("Verify PASSED!");
    } else {
        System.out.println("Verify FAILED!");
    }

    if (isDebugging) {
        System.out.println("\nMatrix A:");
        matrixA.printDistributedRowMatrix();
        System.out.println("\nMatrix B:");
        matrixB.printDistributedRowMatrix();

        System.out.println("\nTransposedMatrix B:");
        // Load DistributedRowMatrix transposedMatrixB
        DistributedRowMatrix transposedMatrixB = new DistributedRowMatrix(transposedMatrixBPaths.get(0),
                CONF_INPUT_DIR, numColsB, numRowsB);
        transposedMatrixB.setConf(conf);
        transposedMatrixB.printDistributedRowMatrix();

        System.out.println("\nMatrix C:");
        matrixC.printDistributedRowMatrix();
        System.out.println("\nMatrix D:");
        matrixD.printDistributedRowMatrix();

        // Print out log files
        printOutput(conf);
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• »