List of usage examples for org.apache.hadoop.fs FileSystem create
public FSDataOutputStream create(Path f) throws IOException
From source file:at.illecker.hama.hybrid.examples.testrootbeer.TestRootbeerHybridBSP.java
License:Apache License
@Override public void bsp(BSPPeer<IntWritable, IntWritable, NullWritable, IntWritable, NullWritable> peer) throws IOException, SyncException, InterruptedException { long startTime = System.currentTimeMillis(); // test input int[] input = new int[CONF_BLOCK_SIZE * CONF_GRID_SIZE]; IntWritable key = new IntWritable(); IntWritable value = new IntWritable(); while (peer.readNext(key, value)) { input[key.get()] = value.get();/* w w w .ja v a 2 s. c om*/ } String peerName = peer.getPeerName(); String[] allPeerNames = peer.getAllPeerNames(); long stopTime = System.currentTimeMillis(); // Debug output BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration()); FileSystem fs = FileSystem.get(peer.getConfiguration()); FSDataOutputStream outStream = fs .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log")); outStream.writeChars("TestRootbeerHybridBSP.bsp executed on CPU!\n"); outStream.writeChars("TestRootbeerHybridBSP,CPUTime=" + (stopTime - startTime) + " ms\n"); outStream.writeChars("TestRootbeerHybridBSP,CPUTime=" + ((stopTime - startTime) / 1000.0) + " seconds\n"); outStream.writeChars("TestRootbeerHybridBSP,peerName: '" + peerName + "'\n"); outStream.writeChars("TestRootbeerHybridBSP,getAllPeerNames: '" + Arrays.toString(allPeerNames) + "'\n"); // outStream.writeChars("TestRootbeerHybridBSP,input: '" // + Arrays.toString(input) + "'\n"); // Verify input peer.reopenInput(); while (peer.readNext(key, value)) { Assert.assertEquals(value.get(), input[key.get()]); } outStream.writeChars("TestRootbeerHybridBSP.bsp: input verified!'\n"); outStream.close(); }
From source file:at.illecker.hama.hybrid.examples.testrootbeer.TestRootbeerHybridBSP.java
License:Apache License
@Override public void bspGpu(BSPPeer<IntWritable, IntWritable, NullWritable, IntWritable, NullWritable> peer, Rootbeer rootbeer) throws IOException, SyncException, InterruptedException { TestRootbeerKernel kernel = new TestRootbeerKernel(CONF_BLOCK_SIZE * CONF_GRID_SIZE); // Run GPU Kernels Context context = rootbeer.createDefaultContext(); Stopwatch watch = new Stopwatch(); watch.start();/* w ww . ja v a 2 s .c o m*/ rootbeer.run(kernel, new ThreadConfig(CONF_BLOCK_SIZE, CONF_GRID_SIZE, CONF_BLOCK_SIZE * CONF_GRID_SIZE), context); watch.stop(); // Debug output BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration()); FileSystem fs = FileSystem.get(peer.getConfiguration()); FSDataOutputStream outStream = fs .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log")); outStream.writeChars("TestRootbeerHybridBSP.bspGpu executed on GPU!\n"); List<StatsRow> stats = context.getStats(); for (StatsRow row : stats) { outStream.writeChars(" StatsRow:\n"); outStream.writeChars(" serial time: " + row.getSerializationTime() + "\n"); outStream.writeChars(" exec time: " + row.getExecutionTime() + "\n"); outStream.writeChars(" deserial time: " + row.getDeserializationTime() + "\n"); outStream.writeChars(" num blocks: " + row.getNumBlocks() + "\n"); outStream.writeChars(" num threads: " + row.getNumThreads() + "\n"); } outStream.writeChars("TestRootbeerHybridBSP,GPUTime=" + watch.elapsedTimeMillis() + " ms\n"); outStream .writeChars("TestRootbeerHybridBSP,GPUTime=" + (watch.elapsedTimeMillis() / 1000.0) + " seconds\n"); outStream.writeChars("TestRootbeerHybridBSP,peerName: '" + kernel.peerName + "'\n"); outStream.writeChars( "TestRootbeerHybridBSP,getAllPeerNames: '" + Arrays.toString(kernel.allPeerNames) + "'\n"); // outStream.writeChars("TestRootbeerHybridBSP,input: '" // + Arrays.toString(kernel.input) + "'\n"); // Verify input peer.reopenInput(); IntWritable key = new IntWritable(); IntWritable value = new IntWritable(); while (peer.readNext(key, value)) { Assert.assertEquals(value.get(), kernel.input[key.get()]); } outStream.writeChars("TestRootbeerHybridBSP.bspGpu: input verified!'\n"); outStream.close(); }
From source file:at.illecker.hama.rootbeer.examples.hellorootbeer.HelloRootbeerGpuBSP.java
License:Apache License
@Override public void bsp(BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, DoubleWritable> peer) throws IOException, SyncException, InterruptedException { // Generate GPU Kernels List<Kernel> kernels = new ArrayList<Kernel>(); for (int i = 0; i < m_kernelCount; i++) { kernels.add(new HelloRootbeerKernel(m_iterations)); }//from w w w. j ava 2 s.com // Run GPU Kernels Rootbeer rootbeer = new Rootbeer(); Context context = rootbeer.createDefaultContext(); Stopwatch watch = new Stopwatch(); watch.start(); rootbeer.run(kernels, context); watch.stop(); // Write log to dfs BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration()); FileSystem fs = FileSystem.get(peer.getConfiguration()); FSDataOutputStream outStream = fs .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log")); outStream.writeChars("KernelCount: " + m_kernelCount + "\n"); outStream.writeChars("Iterations: " + m_iterations + "\n"); outStream.writeChars("GPU time: " + watch.elapsedTimeMillis() + " ms\n"); List<StatsRow> stats = context.getStats(); for (StatsRow row : stats) { outStream.writeChars(" StatsRow:\n"); outStream.writeChars(" serial time: " + row.getSerializationTime() + "\n"); outStream.writeChars(" exec time: " + row.getExecutionTime() + "\n"); outStream.writeChars(" deserial time: " + row.getDeserializationTime() + "\n"); outStream.writeChars(" num blocks: " + row.getNumBlocks() + "\n"); outStream.writeChars(" num threads: " + row.getNumThreads() + "\n"); } outStream.close(); // Send result to MasterTask for (int i = 0; i < m_kernelCount; i++) { HelloRootbeerKernel kernel = (HelloRootbeerKernel) kernels.get(i); peer.send(m_masterTask, new DoubleWritable(kernel.m_result)); } peer.sync(); }
From source file:at.illecker.hama.rootbeer.examples.matrixmultiplication.compositeinput.cpu.MatrixMultiplicationBSPCpu.java
License:Apache License
@Override public void setup(BSPPeer<IntWritable, TupleWritable, IntWritable, VectorWritable, MatrixRowMessage> peer) throws IOException { Configuration conf = peer.getConfiguration(); outCardinality = conf.getInt(OUT_CARD, Integer.MAX_VALUE); isDebuggingEnabled = conf.getBoolean(DEBUG, false); // Choose one as a master, who sorts the matrix rows at the end this.masterTask = peer.getPeerName(peer.getNumPeers() / 2); // Init logging if (isDebuggingEnabled) { try {//from w ww . j a v a2s . c o m FileSystem fs = FileSystem.get(conf); logger = fs.create(new Path(FileOutputFormat.getOutputPath(new BSPJob((HamaConfiguration) conf)) + "/BSP_" + peer.getTaskId() + ".log")); logger.writeChars("bsp,setup,outCardinality=" + outCardinality + "\n"); } catch (IOException e) { e.printStackTrace(); } } }
From source file:at.illecker.hama.rootbeer.examples.matrixmultiplication.cpu.MatrixMultiplicationBSPCpu.java
License:Apache License
@Override public void setup( BSPPeer<IntWritable, PipesVectorWritable, IntWritable, PipesVectorWritable, MatrixRowMessage> peer) throws IOException { Configuration conf = peer.getConfiguration(); m_isDebuggingEnabled = conf.getBoolean(CONF_DEBUG, false); // Choose one as a master, who sorts the matrix rows at the end // m_masterTask = peer.getPeerName(peer.getNumPeers() / 2); // TODO//from w w w .j ava2s .c o m // task must be 0 otherwise write out does NOT work! m_masterTask = peer.getPeerName(0); // Init logging if (m_isDebuggingEnabled) { try { FileSystem fs = FileSystem.get(conf); m_logger = fs.create(new Path(FileOutputFormat.getOutputPath(new BSPJob((HamaConfiguration) conf)) + "/BSP_" + peer.getTaskId() + ".log")); } catch (IOException e) { e.printStackTrace(); } } // Receive transposed Matrix B SequenceFile.Reader reader = new SequenceFile.Reader(FileSystem.get(conf), new Path(conf.get(CONF_MATRIX_MULT_B_PATH)), conf); IntWritable bKey = new IntWritable(); PipesVectorWritable bVector = new PipesVectorWritable(); // for each col of matrix B (cause by transposed B) while (reader.next(bKey, bVector)) { m_bColumns.add(new KeyValuePair<Integer, DoubleVector>(bKey.get(), bVector.getVector())); if (m_isDebuggingEnabled) { m_logger.writeChars("setup,read,transposedMatrixB,key=" + bKey.get() + ",value=" + bVector.getVector().toString() + "\n"); } } reader.close(); }
From source file:at.illecker.hama.rootbeer.examples.matrixmultiplication.gpu.MatrixMultiplicationBSPGpu.java
License:Apache License
@Override public void setup(BSPPeer<IntWritable, VectorWritable, IntWritable, VectorWritable, NullWritable> peer) throws IOException { Configuration conf = peer.getConfiguration(); m_isDebuggingEnabled = conf.getBoolean(CONF_DEBUG, false); // Choose one as a master, who sorts the matrix rows at the end // m_masterTask = peer.getPeerName(peer.getNumPeers() / 2); // TODO/*from ww w . j a v a2s.c o m*/ // task must be 0 otherwise write out does NOT work! m_masterTask = peer.getPeerName(0); this.m_blockSize = Integer.parseInt(peer.getConfiguration().get(CONF_BLOCKSIZE)); this.m_gridSize = Integer.parseInt(peer.getConfiguration().get(CONF_GRIDSIZE)); // Init logging if (m_isDebuggingEnabled) { try { FileSystem fs = FileSystem.get(conf); m_logger = fs.create(new Path(FileOutputFormat.getOutputPath(new BSPJob((HamaConfiguration) conf)) + "/BSP_" + peer.getTaskId() + ".log")); } catch (IOException e) { e.printStackTrace(); } } // Load matrixB SequenceFile.Reader reader = new SequenceFile.Reader(FileSystem.get(conf), new Path(conf.get(CONF_MATRIX_MULT_B_PATH)), conf); List<DoubleVector> matrixB = new ArrayList<DoubleVector>(); IntWritable bKey = new IntWritable(); VectorWritable bVector = new VectorWritable(); // for each row of matrix B while (reader.next(bKey, bVector)) { matrixB.add(bVector.getVector()); if (m_isDebuggingEnabled) { m_logger.writeChars("bsp,setup,MatrixB (" + bKey.get() + "," + bVector.getVector() + ")\n"); } } reader.close(); // Convert matrixB to double array for GPU kernels m_matrixBArr = toArray(matrixB); if (m_isDebuggingEnabled) { for (int i = 0; i < m_matrixBArr.length; i++) { m_logger.writeChars("bsp,setup,MatrixBArr (" + i + "," + Arrays.toString(m_matrixBArr[i]) + ")\n"); } } // threadSliceSize defines how much multipliers // of column B has to be multiplied with column A m_threadSliceSize = divup(m_matrixBArr.length, m_blockSize); // blockSliceSize defines the column slice amount // columns of B per blockIters m_blockSliceSize = divup(m_matrixBArr[0].length, m_gridSize); if (m_isDebuggingEnabled) { m_logger.writeChars("bsp,setup,blockSize=" + m_blockSize + ",gridSize=" + m_gridSize + ",threadSliceSize=" + m_threadSliceSize + ",blockSliceSize=" + m_blockSliceSize + "\n"); } }
From source file:at.illecker.hama.rootbeer.examples.piestimator.cpu.PiEstimatorCpuBSP.java
License:Apache License
@Override public void cleanup(BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, LongWritable> peer) throws IOException { // MasterTask writes out results if (peer.getPeerName().equals(m_masterTask)) { int numMessages = peer.getNumCurrentMessages(); long totalHits = 0; LongWritable received;/* w w w . java2s . c om*/ while ((received = peer.getCurrentMessage()) != null) { totalHits += received.get(); } double pi = 4.0 * totalHits / (m_calculationsPerBspTask * numMessages); // DEBUG if (m_isDebuggingEnabled) { // Write log to dfs BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration()); FileSystem fs = FileSystem.get(peer.getConfiguration()); FSDataOutputStream outStream = fs .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log")); outStream.writeChars("BSP=PiEstimatorCpuBSP,Iterations=" + m_iterations + "\n"); outStream.writeChars("totalHits: " + totalHits + "\n"); outStream.writeChars("numMessages: " + numMessages + "\n"); outStream.writeChars("calculationsPerBspTask: " + m_calculationsPerBspTask + "\n"); outStream.writeChars("calculationsTotal: " + (m_calculationsPerBspTask * numMessages) + "\n"); outStream.close(); } peer.write( new Text("Estimated value of PI(3,14159265) using " + (m_calculationsPerBspTask * numMessages) // + (peer.getNumPeers() * m_threadCount * m_iterations) + " points is"), new DoubleWritable(pi)); } }
From source file:at.illecker.hama.rootbeer.examples.piestimator.gpu.PiEstimatorGpuBSP.java
License:Apache License
@Override public void bsp(BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, LongWritable> peer) throws IOException, SyncException, InterruptedException { PiEstimatorKernel kernel = new PiEstimatorKernel(m_calculationsPerThread, System.currentTimeMillis()); // Run GPU Kernels Rootbeer rootbeer = new Rootbeer(); Context context = rootbeer.createDefaultContext(); Stopwatch watch = new Stopwatch(); watch.start();/* w w w. j ava 2s . co m*/ rootbeer.run(kernel, new ThreadConfig(m_blockSize, m_gridSize, m_blockSize * m_gridSize), context); watch.stop(); // Get GPU results long totalHits = 0; Result[] resultList = kernel.resultList.getList(); for (Result result : resultList) { if (result == null) { // break at end of list break; } totalHits += result.hits; } // DEBUG if (m_isDebuggingEnabled) { // Write log to dfs BSPJob job = new BSPJob((HamaConfiguration) peer.getConfiguration()); FileSystem fs = FileSystem.get(peer.getConfiguration()); FSDataOutputStream outStream = fs .create(new Path(FileOutputFormat.getOutputPath(job), peer.getTaskId() + ".log")); outStream.writeChars("BSP=PiEstimatorGpuBSP,Iterations=" + m_iterations + ",GPUTime=" + watch.elapsedTimeMillis() + "ms\n"); List<StatsRow> stats = context.getStats(); for (StatsRow row : stats) { outStream.writeChars(" StatsRow:\n"); outStream.writeChars(" serial time: " + row.getSerializationTime() + "\n"); outStream.writeChars(" exec time: " + row.getExecutionTime() + "\n"); outStream.writeChars(" deserial time: " + row.getDeserializationTime() + "\n"); outStream.writeChars(" num blocks: " + row.getNumBlocks() + "\n"); outStream.writeChars(" num threads: " + row.getNumThreads() + "\n"); } outStream.writeChars("totalHits: " + totalHits + "\n"); outStream.writeChars("calculationsPerThread: " + m_calculationsPerThread + "\n"); outStream.writeChars("calculationsTotal: " + m_calculationsPerThread * m_blockSize * m_gridSize + "\n"); outStream.close(); } // Send result to MasterTask peer.send(m_masterTask, new LongWritable(totalHits)); peer.sync(); }
From source file:avro.HadoopAvro.java
License:Open Source License
private void createAvroFile() throws IOException { Path inputPath = new Path(INPUT_PATH); FileSystem fs = FileSystem.get(new Configuration()); fs.delete(inputPath, true);/*from w ww . ja v a 2 s . com*/ DataFileWriter<User> fileWriter = new DataFileWriter<>(new GenericDatumWriter<User>(User.SCHEMA)); fileWriter.create(User.SCHEMA, fs.create(new Path(inputPath, "file.avro"))); IntStream.range(0, 100).mapToObj(i -> new User("name" + i, "pass" + i, i, i % 2 == 0)) .forEach(user -> Util.uncheckRun(() -> fileWriter.append(user))); fileWriter.close(); fs.close(); }
From source file:azkaban.jobtype.javautils.HadoopUtils.java
License:Apache License
public static void saveProps(FileSystem fs, Props props, String file) throws IOException { Path path = new Path(file); // create directory if it does not exist. Path parent = path.getParent(); if (!fs.exists(parent)) fs.mkdirs(parent);/*w w w. j a v a 2s . c o m*/ // write out properties OutputStream output = fs.create(path); try { props.storeFlattened(output); } finally { output.close(); } }