Example usage for org.apache.hadoop.fs FileSystem create

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem create.

Prototype

public FSDataOutputStream create(Path f) throws IOException 

Source Link

Document

Create an FSDataOutputStream at the indicated Path.

Usage

From source file:at.illecker.hama.hybrid.examples.matrixmultiplication.MatrixMultiplicationHybridBSP.java

License:Apache License

/********************************* GPU *********************************/
@Override/*from  www  .ja  v  a  2  s .  c  om*/
public void setupGpu(
        BSPPeer<IntWritable, PipesVectorWritable, IntWritable, PipesVectorWritable, MatrixRowMessage> peer)
        throws IOException, SyncException, InterruptedException {

    HamaConfiguration conf = peer.getConfiguration();
    this.m_isDebuggingEnabled = conf.getBoolean(CONF_DEBUG, false);

    // Choose one as a master, who sorts the matrix rows at the end
    // m_masterTask = peer.getPeerName(peer.getNumPeers() / 2);

    // TODO
    // task must be 0 otherwise write out does NOT work!
    this.m_masterTask = peer.getPeerName(0);

    this.m_blockSize = Integer.parseInt(peer.getConfiguration().get(CONF_BLOCKSIZE));

    this.m_gridSize = Integer.parseInt(peer.getConfiguration().get(CONF_GRIDSIZE));

    // Init logging
    if (m_isDebuggingEnabled) {
        try {
            FileSystem fs = FileSystem.get(conf);
            m_logger = fs.create(new Path(FileOutputFormat.getOutputPath(new BSPJob((HamaConfiguration) conf))
                    + "/BSP_" + peer.getTaskId() + ".log"));

        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

---

From source file:at.illecker.hama.hybrid.examples.matrixmultiplication2.MatrixMultiplicationHybridBSP.java

License:Apache License

/********************************* CPU *********************************/
@Override//from  w w w . j av a 2  s  . co  m
public void setup(BSPPeer<IntWritable, VectorWritable, IntWritable, VectorWritable, MatrixRowMessage> peer)
        throws IOException {

    HamaConfiguration conf = peer.getConfiguration();
    this.m_isDebuggingEnabled = conf.getBoolean(CONF_DEBUG, false);

    // used by GPU only
    m_tileWidth = conf.getInt(CONF_TILE_WIDTH, 32);

    // Choose one as a master, who sorts the matrix rows at the end
    // this.m_masterTask = peer.getPeerName(peer.getNumPeers() / 2);
    // TODO
    // task must be 0 otherwise write out does NOT work!
    this.m_masterTask = peer.getPeerName(0);

    // Init logging
    if (m_isDebuggingEnabled) {
        try {
            FileSystem fs = FileSystem.get(conf);
            m_logger = fs.create(new Path(FileOutputFormat.getOutputPath(new BSPJob((HamaConfiguration) conf))
                    + "/BSP_" + peer.getTaskId() + ".log"));

        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    // Load transposed Matrix B
    SequenceFile.Reader reader = new SequenceFile.Reader(FileSystem.get(conf),
            new Path(conf.get(CONF_MATRIX_B_PATH)), conf);

    IntWritable transposedMatrixBRowId = new IntWritable();
    VectorWritable transposedMatrixBRow = new VectorWritable();

    // for each col of matrix B (cause by transposed B)
    while (reader.next(transposedMatrixBRowId, transposedMatrixBRow)) {
        m_transposedMatrixB.add(new KeyValuePair<Integer, DoubleVector>(transposedMatrixBRowId.get(),
                transposedMatrixBRow.getVector()));
        if (m_isDebuggingEnabled) {
            m_logger.writeChars("setup,read,transposedMatrixB,key=" + transposedMatrixBRowId.get() + ",value="
                    + transposedMatrixBRow.getVector().toString() + "\n");
        }
    }
    reader.close();
}

---

From source file:at.illecker.hama.hybrid.examples.onlinecf.OnlineCFTrainHybridBSP.java

License:Apache License

/********************************* CPU *********************************/
// **********************************************************************
// setup//from  w  w w .j  ava  2 s.c  om
// **********************************************************************
@Override
public void setup(BSPPeer<LongWritable, PipesVectorWritable, Text, PipesVectorWritable, ItemMessage> peer)
        throws IOException {

    long startTime = System.currentTimeMillis();

    this.m_conf = peer.getConfiguration();
    this.m_isDebuggingEnabled = m_conf.getBoolean(CONF_DEBUG, false);

    this.m_maxIterations = m_conf.getInt(OnlineCF.CONF_ITERATION_COUNT, OnlineCF.DFLT_ITERATION_COUNT);

    this.m_matrixRank = m_conf.getInt(OnlineCF.CONF_MATRIX_RANK, OnlineCF.DFLT_MATRIX_RANK);

    this.m_skipCount = m_conf.getInt(OnlineCF.CONF_SKIP_COUNT, OnlineCF.DFLT_SKIP_COUNT);

    // Init logging
    if (m_isDebuggingEnabled) {
        try {
            FileSystem fs = FileSystem.get(m_conf);
            m_logger = fs.create(new Path(FileOutputFormat.getOutputPath(new BSPJob((HamaConfiguration) m_conf))
                    + "/BSP_" + peer.getTaskId() + ".log"));

        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    this.m_setupTimeCpu = System.currentTimeMillis() - startTime;
}

---

From source file:at.illecker.hama.hybrid.examples.onlinecf.OnlineCFTrainHybridBSP.java

License:Apache License

/********************************* GPU *********************************/
// **********************************************************************
// setupGpu//w  w  w .  jav a 2  s .  c o m
// **********************************************************************
@Override
public void setupGpu(BSPPeer<LongWritable, PipesVectorWritable, Text, PipesVectorWritable, ItemMessage> peer)
        throws IOException, SyncException, InterruptedException {

    long startTime = System.currentTimeMillis();

    this.m_conf = peer.getConfiguration();
    this.m_isDebuggingEnabled = m_conf.getBoolean(CONF_DEBUG, false);

    this.m_maxIterations = m_conf.getInt(OnlineCF.CONF_ITERATION_COUNT, OnlineCF.DFLT_ITERATION_COUNT);

    this.m_matrixRank = m_conf.getInt(OnlineCF.CONF_MATRIX_RANK, OnlineCF.DFLT_MATRIX_RANK);

    this.m_skipCount = m_conf.getInt(OnlineCF.CONF_SKIP_COUNT, OnlineCF.DFLT_SKIP_COUNT);

    this.m_blockSize = Integer.parseInt(this.m_conf.get(CONF_BLOCKSIZE));
    this.m_gridSize = Integer.parseInt(this.m_conf.get(CONF_GRIDSIZE));

    // Init logging
    if (m_isDebuggingEnabled) {
        try {
            FileSystem fs = FileSystem.get(m_conf);
            m_logger = fs.create(new Path(FileOutputFormat.getOutputPath(new BSPJob((HamaConfiguration) m_conf))
                    + "/BSP_" + peer.getTaskId() + ".log"));

        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    this.m_setupTimeGpu = System.currentTimeMillis() - startTime;
}

---

From source file:at.illecker.hama.hybrid.examples.piestimator.PiEstimatorHybridBSP.java

License:Apache License

/********************************* CPU *********************************/
@Override//from w  w w.  j a va  2 s .co  m
public void setup(BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, LongWritable> peer)
        throws IOException {

    m_conf = peer.getConfiguration();
    this.m_timeMeasurement = m_conf.getBoolean(CONF_TIME, false);
    this.m_isDebuggingEnabled = m_conf.getBoolean(CONF_DEBUG, false);

    // Init logging
    if (m_isDebuggingEnabled) {
        try {
            FileSystem fs = FileSystem.get(m_conf);
            m_logger = fs.create(new Path(FileOutputFormat.getOutputPath(new BSPJob((HamaConfiguration) m_conf))
                    + "/BSP_" + peer.getTaskId() + ".log"));

        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    long startTime = 0;
    if (m_timeMeasurement) {
        startTime = System.currentTimeMillis();
    }

    int totalTaskNum = peer.getNumPeers();
    int cpuTaskNum = totalTaskNum - m_conf.getInt("bsp.peers.gpu.num", 0);

    // Choose one as a master
    this.m_masterTask = peer.getPeerName(totalTaskNum / 2);

    this.m_iterations = Long.parseLong(m_conf.get(CONF_ITERATIONS));
    this.m_GPUPercentage = Integer.parseInt(m_conf.get(CONF_GPU_PERCENTAGE));
    this.m_blockSize = Integer.parseInt(m_conf.get(CONF_BLOCKSIZE));
    this.m_gridSize = Integer.parseInt(m_conf.get(CONF_GRIDSIZE));

    // Compute work distributions
    long iterationsPerGPUTask = 0;
    long iterationsPerCPU = 0;
    if ((m_GPUPercentage > 0) && (m_GPUPercentage <= 100)) {
        iterationsPerGPUTask = (m_iterations * m_GPUPercentage) / 100;
        iterationsPerCPU = m_iterations - iterationsPerGPUTask;
    } else {
        iterationsPerCPU = m_iterations;
    }
    if (cpuTaskNum > 0) {
        m_iterationsPerCPUTask = iterationsPerCPU / cpuTaskNum;
    }
    m_iterationsPerGPUThread = iterationsPerGPUTask / (m_blockSize * m_gridSize);

    long stopTime = 0;
    if (m_timeMeasurement) {
        stopTime = System.currentTimeMillis();
        LOG.info("# setupTime: " + ((stopTime - startTime) / 1000.0) + " sec");
    }

    // DEBUG
    if (m_isDebuggingEnabled) {
        m_logger.writeChars("PiEstimatorHybrid,GPUPercentage=" + m_GPUPercentage + "\n");
        m_logger.writeChars("PiEstimatorHybrid,cpuTaskNum=" + cpuTaskNum + "\n");
        m_logger.writeChars("PiEstimatorHybrid,iterationsPerCPU=" + iterationsPerCPU + "\n");
        m_logger.writeChars("PiEstimatorHybrid,iterationsPerCPUTask=" + m_iterationsPerCPUTask + "\n");
        m_logger.writeChars("PiEstimatorHybrid,iterationsPerGPUTask=" + iterationsPerGPUTask + "\n");
        m_logger.writeChars("PiEstimatorHybrid,iterationsPerGPUThread=" + m_iterationsPerGPUThread + "\n");
        if (m_timeMeasurement) {
            m_logger.writeChars(
                    "PiEstimatorHybrid,setupTime: " + ((stopTime - startTime) / 1000.0) + " sec\n\n");
        }
    }

}

---

From source file:at.illecker.hama.hybrid.examples.summation.SummationBSP.java

License:Apache License

@Override
public void bsp(BSPPeer<Text, Text, Text, DoubleWritable, DoubleWritable> peer)
        throws IOException, SyncException, InterruptedException {

    BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration());
    FileSystem fs = FileSystem.get(peer.getConfiguration());
    FSDataOutputStream outStream = fs
            .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log"));

    outStream.writeChars("SummationBSP.bsp executed on CPU!\n");

    double intermediateSum = 0.0;
    Text key = new Text();
    Text value = new Text();

    while (peer.readNext(key, value)) {
        outStream.writeChars("SummationBSP.bsp key: " + key + " value: " + value + "\n");
        intermediateSum += Double.parseDouble(value.toString());
    }//from www .  ja  v  a2s . c  o  m

    outStream.writeChars("SummationBSP.bsp send intermediateSum: " + intermediateSum + "\n");

    peer.send(m_masterTask, new DoubleWritable(intermediateSum));
    peer.sync();

    // Consume messages
    if (peer.getPeerName().equals(m_masterTask)) {
        outStream.writeChars("SummationBSP.bsp consume messages...\n");

        double sum = 0.0;
        int msg_count = peer.getNumCurrentMessages();

        for (int i = 0; i < msg_count; i++) {
            DoubleWritable msg = peer.getCurrentMessage();
            outStream.writeChars("SummationBSP.bsp message: " + msg.get() + "\n");
            sum += msg.get();
        }

        outStream.writeChars("SummationBSP.bsp write Sum: " + sum + "\n");
        peer.write(new Text("Sum"), new DoubleWritable(sum));
    }
    outStream.close();
}

---

From source file:at.illecker.hama.hybrid.examples.summation.SummationBSP.java

License:Apache License

@Override
public void bspGpu(BSPPeer<Text, Text, Text, DoubleWritable, DoubleWritable> peer, Rootbeer rootbeer)
        throws IOException, SyncException, InterruptedException {

    BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration());
    FileSystem fs = FileSystem.get(peer.getConfiguration());
    FSDataOutputStream outStream = fs
            .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log"));

    outStream.writeChars("SummationBSP.bspGpu executed on GPU!\n");

    SummationKernel kernel = new SummationKernel(m_masterTask);

    // Run GPU Kernels
    Context context = rootbeer.createDefaultContext();
    Stopwatch watch = new Stopwatch();
    watch.start();//from w ww.j  a  va  2 s .  c  o  m
    // 1 Kernel within 1 Block
    rootbeer.run(kernel, new ThreadConfig(1, 1, 1), context);
    watch.stop();

    List<StatsRow> stats = context.getStats();
    for (StatsRow row : stats) {
        outStream.writeChars("  StatsRow:\n");
        outStream.writeChars("    serial time: " + row.getSerializationTime() + "\n");
        outStream.writeChars("    exec time: " + row.getExecutionTime() + "\n");
        outStream.writeChars("    deserial time: " + row.getDeserializationTime() + "\n");
        outStream.writeChars("    num blocks: " + row.getNumBlocks() + "\n");
        outStream.writeChars("    num threads: " + row.getNumThreads() + "\n");
    }

    outStream.writeChars("SummationBSP,GPUTime=" + watch.elapsedTimeMillis() + "ms\n");
    outStream.writeChars("SummationBSP,peerName: '" + kernel.m_peerName + "'\n");
    outStream.writeChars("SummationBSP,numPeers: '" + kernel.m_numPeers + "'\n");
    outStream.close();
}

---

From source file:at.illecker.hama.hybrid.examples.summation.SummationBSP.java

License:Apache License

static BigDecimal writeSummationInputFile(FileSystem fs, Path dir, int fileCount) throws IOException {

    BigDecimal sum = new BigDecimal(0);
    Random rand = new Random();
    double rangeMin = 0;
    double rangeMax = 100;

    for (int i = 0; i < fileCount; i++) {
        DataOutputStream out = fs.create(new Path(dir, "part" + i));

        // loop between 50 and 149 times
        for (int j = 0; j < rand.nextInt(100) + 50; j++) {
            // generate key value pair inputs
            double randomValue = rangeMin + (rangeMax - rangeMin) * rand.nextDouble();

            String truncatedValue = new BigDecimal(randomValue)
                    .setScale(DOUBLE_PRECISION, BigDecimal.ROUND_DOWN).toString();

            String line = "key" + (j + 1) + "\t" + truncatedValue + "\n";
            out.writeBytes(line);/*ww w  .  j av a  2 s . c om*/

            sum = sum.add(new BigDecimal(truncatedValue));
            LOG.debug("input[" + j + "]: '" + line + "' sum: " + sum.toString());
        }
        out.close();
    }
    return sum;
}

---

From source file:at.illecker.hama.hybrid.examples.testglobalgpusync.TestGlobalGpuSyncHybridBSP.java

License:Apache License

@Override
public void bsp(BSPPeer<NullWritable, NullWritable, NullWritable, NullWritable, IntWritable> peer)
        throws IOException, SyncException, InterruptedException {

    // Debug output
    HamaConfiguration conf = peer.getConfiguration();
    FileSystem fs = FileSystem.get(peer.getConfiguration());
    FSDataOutputStream outStream = fs.create(new Path(conf.get(CONF_TMP_DIR), peer.getTaskId() + ".log"));
    outStream.writeChars("TestGlobalGpuSycHybridBSP.bsp executed on CPU!\n");

    peer.send(m_masterTask, new IntWritable(peer.getPeerIndex()));

    peer.sync();// w w  w. ja v a 2s .c  o  m

    // If master, fetch messages
    if (peer.getPeerName().equals(m_masterTask)) {
        peer.getNumCurrentMessages();

        int msgCount = peer.getNumCurrentMessages();
        for (int i = 0; i < msgCount; i++) {
            int id = peer.getCurrentMessage().get();
            outStream.writeChars(id + "\n");
        }
    }

    outStream.close();
}

---

From source file:at.illecker.hama.hybrid.examples.testglobalgpusync.TestGlobalGpuSyncHybridBSP.java

License:Apache License

@Override
public void bspGpu(BSPPeer<NullWritable, NullWritable, NullWritable, NullWritable, IntWritable> peer,
        Rootbeer rootbeer) throws IOException, SyncException, InterruptedException {

    HamaConfiguration conf = peer.getConfiguration();
    int blockSize = Integer.parseInt(conf.get(CONF_BLOCK_SIZE));
    int gridSize = Integer.parseInt(conf.get(CONF_GRID_SIZE));

    TestGlobalGpuSyncKernel kernel = new TestGlobalGpuSyncKernel(m_masterTask);

    // Run GPU Kernels
    Context context = rootbeer.createDefaultContext();
    Stopwatch watch = new Stopwatch();
    watch.start();//from w w w . j  a  v a2  s.  c om
    rootbeer.run(kernel, new ThreadConfig(blockSize, gridSize, blockSize * gridSize), context);
    watch.stop();

    // Debug output
    FileSystem fs = FileSystem.get(conf);
    FSDataOutputStream outStream = fs.create(new Path(conf.get(CONF_TMP_DIR), peer.getTaskId() + ".log"));

    outStream.writeChars("TestGlobalGpuSycHybridBSP.bspGpu executed on GPU!\n");
    List<StatsRow> stats = context.getStats();
    for (StatsRow row : stats) {
        outStream.writeChars("  StatsRow:\n");
        outStream.writeChars("    serial time: " + row.getSerializationTime() + "\n");
        outStream.writeChars("    exec time: " + row.getExecutionTime() + "\n");
        outStream.writeChars("    deserial time: " + row.getDeserializationTime() + "\n");
        outStream.writeChars("    num blocks: " + row.getNumBlocks() + "\n");
        outStream.writeChars("    num threads: " + row.getNumThreads() + "\n");
    }

    outStream.writeChars("TestGlobalGpuSycHybridBSP,GPUTime=" + watch.elapsedTimeMillis() + " ms\n");
    outStream.writeChars("TestGlobalGpuSycHybridBSP,BlockSize=" + blockSize + "\n");
    outStream.writeChars("TestGlobalGpuSycHybridBSP,GridSize=" + gridSize + "\n");
    outStream.writeChars("TestGlobalGpuSycHybridBSP,TotalThreads=" + (blockSize * gridSize) + "\n");
    outStream.writeChars("TestGlobalGpuSycHybridBSP,MessageCount=" + kernel.messageCount + "\n");
    outStream.writeChars("TestGlobalGpuSycHybridBSP,MessageSum=" + kernel.messageSum + "\n");

    Assert.assertEquals((blockSize * gridSize), kernel.messageCount);
    int n = (blockSize * gridSize) - 1;
    Assert.assertEquals((n * (n + 1)) / 2, kernel.messageSum);
    outStream.writeChars("TestGlobalGpuSycHybridBSP.bspGpu: messages verified!'\n");
    outStream.close();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• »