Example usage for org.apache.hadoop.conf Configuration Configuration

List of usage examples for org.apache.hadoop.conf Configuration Configuration

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. Configuration

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration Configuration.

Prototype

public Configuration()

Source Link

Document

A new configuration.

Usage

From source file:at.illecker.hadoop.rootbeer.examples.matrixmultiplication.DistributedRowMatrix.java

License:Apache License

/**
 * This implements matrix multiplication A * B using MapReduce tasks on CPU or
 * GPU/*from   ww w.  j a  v  a2s.  co  m*/
 * 
 * @param other a DistributedRowMatrix
 * @param outPath path to write result to
 * @param useGPU use GPU or CPU (default: false, use CPU)
 * @return a DistributedRowMatrix containing the product
 */
public DistributedRowMatrix multiplyMapReduce(DistributedRowMatrix other, Path outPath, boolean useGPU,
        boolean isMatrixATransposed, int tileWidth, boolean isDebugging) throws IOException {
    // Check if cols of MatrixA = rows of MatrixB
    // (l x m) * (m x n) = (l x n)
    if (numCols != other.numRows()) {
        throw new CardinalityException(numCols, other.numRows());
    }

    Configuration initialConf = (getConf() == null) ? new Configuration() : getConf();

    // Transpose Matrix within a new MapReduce Job
    DistributedRowMatrix transposed = this;
    if (!isMatrixATransposed) {
        transposed = transposed.transpose();
    }
    // Debug
    // System.out.println("DistributedRowMatrix transposed:");
    // transposed.printDistributedRowMatrix();

    // Build MatrixMultiplication job configuration
    Configuration conf = null;
    if (!useGPU) {
        conf = MatrixMultiplicationCpu.createMatrixMultiplicationCpuConf(initialConf, transposed.rowPath,
                other.rowPath, outPath, other.numCols, isDebugging);
    } else { // use GPU
        conf = MatrixMultiplicationGpu.createMatrixMultiplicationGpuConf(initialConf, transposed.rowPath,
                other.rowPath, outPath, other.numCols, tileWidth, isDebugging);
    }

    // Multiply Matrix with transposed one
    JobClient.runJob(new JobConf(conf));

    // Read resulting Matrix from HDFS
    DistributedRowMatrix out = new DistributedRowMatrix(outPath, outputTmpPath, this.numRows, other.numCols());
    out.setConf(conf);

    return out;
}

From source file:at.illecker.hadoop.rootbeer.examples.matrixmultiplication.DistributedRowMatrix.java

License:Apache License

/**
 * Returns the column-wise mean of a DistributedRowMatrix
 * //  w  ww  .  j  a v a 2s. co  m
 * @param vectorClass desired class for the column-wise mean vector e.g.
 *          RandomAccessSparseVector, DenseVector
 * @return Vector containing the column-wise mean of this
 */
public Vector columnMeans(String vectorClass) throws IOException {
    Path outputVectorTmpPath = new Path(outputTmpBasePath, new Path(Long.toString(System.nanoTime())));
    Configuration initialConf = getConf() == null ? new Configuration() : getConf();
    String vectorClassFull = "org.apache.mahout.math." + vectorClass;
    Vector mean = MatrixColumnMeansJob.run(initialConf, rowPath, outputVectorTmpPath, vectorClassFull);
    if (!keepTempFiles) {
        FileSystem fs = outputVectorTmpPath.getFileSystem(conf);
        fs.delete(outputVectorTmpPath, true);
    }
    return mean;
}

From source file:at.illecker.hadoop.rootbeer.examples.matrixmultiplication.DistributedRowMatrix.java

License:Apache License

public DistributedRowMatrix transpose() throws IOException {
    Path outputPath = new Path(outputTmpBasePath, "transpose-" + (System.nanoTime() & 0xFF));
    Configuration initialConf = getConf() == null ? new Configuration() : getConf();
    Configuration conf = TransposeJob.buildTransposeJobConf(initialConf, rowPath, outputPath, numRows);
    JobClient.runJob(new JobConf(conf));
    DistributedRowMatrix m = new DistributedRowMatrix(outputPath, outputTmpPath, numCols, numRows);
    m.setConf(this.conf);
    return m;//from   ww  w .  ja  v a 2  s.  co m
}

From source file:at.illecker.hadoop.rootbeer.examples.matrixmultiplication.DistributedRowMatrix.java

License:Apache License

@Override
public Vector times(Vector v) {
    try {/*w w w .  j  a va  2s.  c  o m*/
        Configuration initialConf = getConf() == null ? new Configuration() : getConf();
        Path outputVectorTmpPath = new Path(outputTmpBasePath, new Path(Long.toString(System.nanoTime())));
        Configuration conf = TimesSquaredJob.createTimesJobConf(initialConf, v, numRows, rowPath,
                outputVectorTmpPath);
        JobClient.runJob(new JobConf(conf));
        Vector result = TimesSquaredJob.retrieveTimesSquaredOutputVector(conf);
        if (!keepTempFiles) {
            FileSystem fs = outputVectorTmpPath.getFileSystem(conf);
            fs.delete(outputVectorTmpPath, true);
        }
        return result;
    } catch (IOException ioe) {
        throw new IllegalStateException(ioe);
    }
}

From source file:at.illecker.hadoop.rootbeer.examples.matrixmultiplication.DistributedRowMatrix.java

License:Apache License

@Override
public Vector timesSquared(Vector v) {
    try {//from www  .jav  a2s  . co m
        Configuration initialConf = getConf() == null ? new Configuration() : getConf();
        Path outputVectorTmpPath = new Path(outputTmpBasePath, new Path(Long.toString(System.nanoTime())));
        Configuration conf = TimesSquaredJob.createTimesSquaredJobConf(initialConf, v, rowPath,
                outputVectorTmpPath);
        JobClient.runJob(new JobConf(conf));
        Vector result = TimesSquaredJob.retrieveTimesSquaredOutputVector(conf);
        if (!keepTempFiles) {
            FileSystem fs = outputVectorTmpPath.getFileSystem(conf);
            fs.delete(outputVectorTmpPath, true);
        }
        return result;
    } catch (IOException ioe) {
        throw new IllegalStateException(ioe);
    }
}

From source file:at.illecker.hadoop.rootbeer.examples.matrixmultiplication.gpu.MatrixMultiplicationGpu.java

License:Apache License

public static Configuration createMatrixMultiplicationGpuConf(Path aPath, Path bPath, Path outPath,
        int outCardinality, int tileWidth, boolean isDebugging) {

    return createMatrixMultiplicationGpuConf(new Configuration(), aPath, bPath, outPath, outCardinality,
            tileWidth, isDebugging);// w ww  .j a  v  a 2  s.  c  om
}

From source file:at.illecker.hadoop.rootbeer.examples.matrixmultiplication.MatrixMultiplicationBenchmark.java

License:Apache License

@Override
protected void setUp() throws Exception {
    m_conf = new Configuration();

    // Try to load Hadoop configuration
    String HADOOP_HOME = System.getenv("HADOOP_HOME");
    String HADOOP_INSTALL = System.getenv("HADOOP_INSTALL");

    if ((HADOOP_HOME != null) || (HADOOP_INSTALL != null) && (!m_runLocally)) {
        String HADOOP = ((HADOOP_HOME != null) ? HADOOP_HOME : HADOOP_INSTALL);

        m_conf.addResource(new Path(HADOOP, "src/core/core-default.xml"));
        m_conf.addResource(new Path(HADOOP, "src/hdfs/hdfs-default.xml"));
        m_conf.addResource(new Path(HADOOP, "src/mapred/mapred-default.xml"));
        m_conf.addResource(new Path(HADOOP, "conf/core-site.xml"));
        m_conf.addResource(new Path(HADOOP, "conf/hdfs-site.xml"));
        m_conf.addResource(new Path(HADOOP, "conf/mapred-site.xml"));
        // System.out.println("Loaded Hadoop configuration from " + HADOOP);

        try {//from w  w w .j  a  va  2s . co  m
            // Connect to HDFS Filesystem
            FileSystem.get(m_conf);
        } catch (Exception e) {
            // HDFS not reachable run Benchmark locally
            m_conf = new Configuration();
            m_runLocally = true;
        }
    }

    // Create random DistributedRowMatrix and write out transposed
    DistributedRowMatrix.createRandomDistributedRowMatrix(m_conf, n, n, new Random(42L),
            m_transposedMatrixAPath, true);
    DistributedRowMatrix.createRandomDistributedRowMatrix(m_conf, n, n, new Random(), m_matrixBPath, false);

    // Load DistributedRowMatrix A and B
    m_transposedMatrixA = new DistributedRowMatrix(m_transposedMatrixAPath, CONF_INPUT_DIR, n, n);
    m_transposedMatrixA.setConf(m_conf);

    m_matrixB = new DistributedRowMatrix(m_matrixBPath, CONF_INPUT_DIR, n, n);
    m_matrixB.setConf(m_conf);

    // Debug output
    System.out.println("CONF_TMP_DIR: " + CONF_TMP_DIR.toString());
    System.out.println("Benchmark " + n + " x " + n + " matrix on " + type);
}

From source file:at.illecker.hama.hybrid.examples.kmeans.KMeansHybridBenchmark.java

License:Apache License

@Override
protected void setUp() throws Exception {
    m_conf = new Configuration();

    // Try to load Hadoop configuration
    String HADOOP_HOME = System.getenv("HADOOP_HOME");
    String HADOOP_INSTALL = System.getenv("HADOOP_INSTALL");
    if ((HADOOP_HOME != null) || (HADOOP_INSTALL != null) && (!m_runLocally)) {
        String HADOOP = ((HADOOP_HOME != null) ? HADOOP_HOME : HADOOP_INSTALL);

        m_conf.addResource(new Path(HADOOP, "src/core/core-default.xml"));
        m_conf.addResource(new Path(HADOOP, "src/hdfs/hdfs-default.xml"));
        m_conf.addResource(new Path(HADOOP, "src/mapred/mapred-default.xml"));
        m_conf.addResource(new Path(HADOOP, "conf/core-site.xml"));
        m_conf.addResource(new Path(HADOOP, "conf/hdfs-site.xml"));
        m_conf.addResource(new Path(HADOOP, "conf/mapred-site.xml"));
        // System.out.println("Loaded Hadoop configuration from " + HADOOP);

        try {/*from  w  w w. j  ava  2s.  co  m*/
            // Connect to HDFS Filesystem
            FileSystem.get(m_conf);
        } catch (Exception e) {
            // HDFS not reachable run Benchmark locally
            m_conf = new Configuration();
            m_runLocally = true;
        }
        // System.out.println("Run Benchmark local: " + m_runLocally);
    }

    // Try to load Hama configuration
    String HAMA_HOME = System.getenv("HAMA_HOME");
    String HAMA_INSTALL = System.getenv("HAMA_INSTALL");
    if ((HAMA_HOME != null) || (HAMA_INSTALL != null) && (!m_runLocally)) {
        String HAMA = ((HAMA_HOME != null) ? HAMA_HOME : HAMA_INSTALL);

        m_conf.addResource(new Path(HAMA, "conf/hama-default.xml"));
        m_conf.addResource(new Path(HAMA, "conf/hama-site.xml"));
        // System.out.println("Loaded Hama configuration from " + HAMA);
    }

    // Setup KMeans config variables
    m_conf.setBoolean(KMeansHybridBSP.CONF_DEBUG, false);
    m_conf.setBoolean("hama.pipes.logging", false);
    m_conf.setBoolean(KMeansHybridBSP.CONF_TIME, false);

    // Set GPU blockSize and gridSize
    m_conf.set(KMeansHybridBSP.CONF_BLOCKSIZE, "" + BLOCK_SIZE);
    m_conf.set(KMeansHybridBSP.CONF_GRIDSIZE, "" + GRID_SIZE);
    // Set maxIterations for KMeans
    m_conf.setInt(KMeansHybridBSP.CONF_MAX_ITERATIONS, maxIteration);
    // Set n for KMeans
    m_conf.setLong(KMeansHybridBSP.CONF_N, n);
    // Set GPUPercentage
    m_conf.setInt(KMeansHybridBSP.CONF_GPU_PERCENTAGE, GPUWorkload);

    Path centerIn = new Path(CONF_CENTER_DIR, "center_in.seq");
    Path centerOut = new Path(CONF_CENTER_DIR, "center_out.seq");
    m_conf.set(KMeansHybridBSP.CONF_CENTER_IN_PATH, centerIn.toString());
    m_conf.set(KMeansHybridBSP.CONF_CENTER_OUT_PATH, centerOut.toString());

    // CPU vs GPU benchmark
    // Plot 1 and 2
    int numGpuBspTask = 0;
    // if (type == CalcType.GPU) {
    // bspTaskNum = 1;
    // numGpuBspTask = 1;
    // GPUWorkload = 100;
    // }

    // CPU + GPU Hybrid benchmark
    // Plot 3
    if (bspTaskNum == maxTaskNum) {
        numGpuBspTask = 1;
        GPUWorkload = 75;
    } else {
        numGpuBspTask = 0;
    }

    // Set CPU tasks
    m_conf.setInt("bsp.peers.num", bspTaskNum);
    // Set GPU tasks
    m_conf.setInt("bsp.peers.gpu.num", numGpuBspTask);

    // Generate input data
    KMeansHybridBSP.prepareInputData(m_conf, FileSystem.get(m_conf), CONF_INPUT_DIR, centerIn, bspTaskNum,
            numGpuBspTask, n, k, vectorDimension, null, GPUWorkload);

    // Debug output
    // System.out.println("CalcType: " + type);
    System.out.println("CONF_TMP_DIR: " + CONF_TMP_DIR.toString());
    System.out.println("NumBspTask: " + m_conf.getInt("bsp.peers.num", 0) + " NumGpuBspTask: "
            + m_conf.getInt("bsp.peers.gpu.num", 0));
    System.out.println("n: " + n + " k: " + k + " vectorDimension: " + vectorDimension + " maxIteration: "
            + maxIteration + " GPUWorkload: " + GPUWorkload + "%");
}

From source file:at.illecker.hama.hybrid.examples.matrixmultiplication.MatrixMultiplicationHybridBenchmark.java

License:Apache License

@Override
protected void setUp() throws Exception {
    m_conf = new Configuration();

    // Try to load Hadoop configuration
    String HADOOP_HOME = System.getenv("HADOOP_HOME");
    String HADOOP_INSTALL = System.getenv("HADOOP_INSTALL");
    if ((HADOOP_HOME != null) || (HADOOP_INSTALL != null) && (!m_runLocally)) {
        String HADOOP = ((HADOOP_HOME != null) ? HADOOP_HOME : HADOOP_INSTALL);

        m_conf.addResource(new Path(HADOOP, "src/core/core-default.xml"));
        m_conf.addResource(new Path(HADOOP, "src/hdfs/hdfs-default.xml"));
        m_conf.addResource(new Path(HADOOP, "src/mapred/mapred-default.xml"));
        m_conf.addResource(new Path(HADOOP, "conf/core-site.xml"));
        m_conf.addResource(new Path(HADOOP, "conf/hdfs-site.xml"));
        m_conf.addResource(new Path(HADOOP, "conf/mapred-site.xml"));
        // System.out.println("Loaded Hadoop configuration from " + HADOOP);

        try {/*w w  w.j av  a  2s .  c o m*/
            // Connect to HDFS Filesystem
            FileSystem.get(m_conf);
        } catch (Exception e) {
            // HDFS not reachable run Benchmark locally
            m_conf = new Configuration();
            m_runLocally = true;
        }
        // System.out.println("Run Benchmark local: " + m_runLocally);
    }

    // Try to load Hama configuration
    String HAMA_HOME = System.getenv("HAMA_HOME");
    String HAMA_INSTALL = System.getenv("HAMA_INSTALL");
    if ((HAMA_HOME != null) || (HAMA_INSTALL != null) && (!m_runLocally)) {
        String HAMA = ((HAMA_HOME != null) ? HAMA_HOME : HAMA_INSTALL);

        m_conf.addResource(new Path(HAMA, "conf/hama-default.xml"));
        m_conf.addResource(new Path(HAMA, "conf/hama-site.xml"));
        // System.out.println("Loaded Hama configuration from " + HAMA);
    }

    // Setup outputs
    m_OUTPUT_DIR_PATH = new Path(OUTPUT_DIR + "/bench_" + System.currentTimeMillis());
    System.out.println("OUTPUT_DIR_PATH: " + m_OUTPUT_DIR_PATH);

    m_MATRIX_A_PATH = new Path(m_OUTPUT_DIR_PATH + "/MatrixA.seq");
    m_MATRIX_B_PATH = new Path(m_OUTPUT_DIR_PATH + "/MatrixB.seq");
    m_MATRIX_C_PATH = new Path(m_OUTPUT_DIR_PATH + "/MatrixC.seq");
    m_MATRIX_D_PATH = new Path(m_OUTPUT_DIR_PATH + "/MatrixD.seq");

    m_blockSize = MatrixMultiplicationHybridBSP.BLOCK_SIZE;
    m_gridSize = MatrixMultiplicationHybridBSP.GRID_SIZE;

    System.out.println("Benchmark MatrixMultiplication " + type + " [blockSize=" + m_blockSize + ",gridSize="
            + m_gridSize + "] " + n + " x " + n + " matrix");

    // Create random DistributedRowMatrix
    DistributedRowMatrix.createRandomDistributedRowMatrix(m_conf, n, n, new Random(42L), m_MATRIX_A_PATH,
            false);

    DistributedRowMatrix.createRandomDistributedRowMatrix(m_conf, n, n, new Random(1337L), m_MATRIX_B_PATH,
            (type == CalcType.CPU) ? true : false);

    // Load DistributedRowMatrix a and b
    m_matrixA = new DistributedRowMatrix(m_MATRIX_A_PATH, m_OUTPUT_DIR_PATH, n, n);
    m_matrixB = new DistributedRowMatrix(m_MATRIX_B_PATH, m_OUTPUT_DIR_PATH, n, n);
    m_matrixA.setConf(m_conf);
    m_matrixB.setConf(m_conf);
}

From source file:at.illecker.hama.hybrid.examples.matrixmultiplication2.MatrixMultiplicationHybridBenchmark.java

License:Apache License

@Override
protected void setUp() throws Exception {
    m_conf = new Configuration();

    // Try to load Hadoop configuration
    String HADOOP_HOME = System.getenv("HADOOP_HOME");
    String HADOOP_INSTALL = System.getenv("HADOOP_INSTALL");
    if ((HADOOP_HOME != null) || (HADOOP_INSTALL != null) && (!m_runLocally)) {
        String HADOOP = ((HADOOP_HOME != null) ? HADOOP_HOME : HADOOP_INSTALL);

        m_conf.addResource(new Path(HADOOP, "src/core/core-default.xml"));
        m_conf.addResource(new Path(HADOOP, "src/hdfs/hdfs-default.xml"));
        m_conf.addResource(new Path(HADOOP, "src/mapred/mapred-default.xml"));
        m_conf.addResource(new Path(HADOOP, "conf/core-site.xml"));
        m_conf.addResource(new Path(HADOOP, "conf/hdfs-site.xml"));
        m_conf.addResource(new Path(HADOOP, "conf/mapred-site.xml"));
        // System.out.println("Loaded Hadoop configuration from " + HADOOP);

        try {/*from  ww w.  j  a  v a2s .  c o  m*/
            // Connect to HDFS Filesystem
            FileSystem.get(m_conf);
        } catch (Exception e) {
            // HDFS not reachable run Benchmark locally
            m_conf = new Configuration();
            m_runLocally = true;
        }
        // System.out.println("Run Benchmark local: " + m_runLocally);
    }

    // Try to load Hama configuration
    String HAMA_HOME = System.getenv("HAMA_HOME");
    String HAMA_INSTALL = System.getenv("HAMA_INSTALL");
    if ((HAMA_HOME != null) || (HAMA_INSTALL != null) && (!m_runLocally)) {
        String HAMA = ((HAMA_HOME != null) ? HAMA_HOME : HAMA_INSTALL);

        m_conf.addResource(new Path(HAMA, "conf/hama-default.xml"));
        m_conf.addResource(new Path(HAMA, "conf/hama-site.xml"));
        // System.out.println("Loaded Hama configuration from " + HAMA);
    }

    // CPU vs GPU benchmark
    int numGpuBspTask = 0;
    // if (type == CalcType.GPU) {
    // bspTaskNum = 1;
    // numGpuBspTask = 1;
    // GPUWorkload = 100;
    // }

    // CPU + GPU Hybrid benchmark
    if (bspTaskNum == maxTaskNum) {
        numGpuBspTask = 1;
        GPUWorkload = 95;
    } else {
        numGpuBspTask = 0;
    }

    // Set CPU tasks
    m_conf.setInt("bsp.peers.num", bspTaskNum);
    m_numBspTask = bspTaskNum;
    // Set GPU tasks
    m_conf.setInt("bsp.peers.gpu.num", numGpuBspTask);
    m_numGpuBspTask = numGpuBspTask;

    m_conf.setBoolean("hama.pipes.logging", false);

    // Generate input matrix A and transposed matrix B
    prepareInput();

    // Debug output
    // System.out.println("CalcType: " + type);
    System.out.println("CONF_TMP_DIR: " + CONF_TMP_DIR.toString());
    System.out.println("NumBspTask: " + m_conf.getInt("bsp.peers.num", 0) + " NumGpuBspTask: "
            + m_conf.getInt("bsp.peers.gpu.num", 0));
    System.out.println("n: " + n + " GPUWorkload: " + GPUWorkload + "%");
}