List of usage examples for org.apache.hadoop.fs FileSystem create
public FSDataOutputStream create(Path f) throws IOException
From source file:babel.prep.corpus.MultipleXMLLangFileOutputFormat.java
License:Apache License
public RecordWriter<Text, Page> getBaseRecordWriter(final FileSystem fs, JobConf job, String name, final Progressable progress) throws IOException { final Path dumpFile = new Path(FileOutputFormat.getOutputPath(job), name); // Get the old copy out of the way if (fs.exists(dumpFile)) fs.delete(dumpFile, true);/* w ww . j av a2 s . c o m*/ final XMLObjectWriter xmlWriter; try { xmlWriter = new XMLObjectWriter(fs.create(dumpFile), false); } catch (Exception e) { throw new RuntimeException("Failed to instantiate XMLObjectWriter."); } return new RecordWriter<Text, Page>() { public synchronized void write(Text key, Page page) throws IOException { try { xmlWriter.write(page); } catch (XMLStreamException e) { throw new RuntimeException("Error writing page XML."); } } public synchronized void close(Reporter reporter) throws IOException { try { xmlWriter.close(); } catch (XMLStreamException e) { throw new RuntimeException("Error closing XMLObjectWriter."); } } }; }
From source file:be.uantwerpen.adrem.disteclat.PrefixComputerReducer.java
License:Apache License
private void createShortFIsFile(Context context) throws IOException { Path path = new Path(createPath(getJobAbsoluteOutputDir(context), OShortFIs, OShortFIs)); FileSystem fs = path.getFileSystem(context.getConfiguration()); shortFIsOut = new PrintStream(fs.create(path)); }
From source file:be.ugent.intec.halvade.utils.ChromosomeSplitter.java
License:Open Source License
public void exportSplitter(String filename, Configuration conf) throws URISyntaxException, IOException { DataOutputStream dos = null;/*from w w w . ja v a 2 s. co m*/ FileSystem hdfs = null; try { hdfs = FileSystem.get(new URI(filename), conf); Path file = new Path(filename); if (hdfs.exists(file)) { hdfs.delete(file, true); } OutputStream os = hdfs.create(file); dos = new DataOutputStream(os); dos.writeInt(regions.size()); for (BedRegion region : regions) { dos.writeUTF(region.contig); dos.writeInt(region.start); dos.writeInt(region.end); dos.writeInt(region.key); } } finally { if (dos != null) dos.close(); } }
From source file:boa.aggregators.MLAggregator.java
License:Apache License
public void saveModel(Object model) { FSDataOutputStream out = null;// w w w.j av a 2 s .c om FileSystem fileSystem = null; Path filePath = null; try { JobContext context = (JobContext) getContext(); Configuration configuration = context.getConfiguration(); int boaJobId = configuration.getInt("boa.hadoop.jobid", 0); JobConf job = new JobConf(configuration); Path outputPath = FileOutputFormat.getOutputPath(job); fileSystem = outputPath.getFileSystem(context.getConfiguration()); fileSystem.mkdirs(new Path("/boa", new Path("" + boaJobId))); filePath = new Path("/boa", new Path("" + boaJobId, new Path(("" + getKey()).split("\\[")[0] + "ML.model"))); if (fileSystem.exists(filePath)) return; out = fileSystem.create(filePath); ByteArrayOutputStream byteOutStream = new ByteArrayOutputStream(); ObjectOutputStream objectOut = new ObjectOutputStream(byteOutStream); objectOut.writeObject(model); objectOut.close(); byte[] serializedObject = byteOutStream.toByteArray(); out.write(serializedObject, 0, serializedObject.length); this.collect(filePath.toString()); } catch (Exception e) { e.printStackTrace(); } finally { try { if (out != null) out.close(); } catch (final Exception e) { e.printStackTrace(); } } }
From source file:boa.io.BoaOutputCommitter.java
License:Apache License
private void storeOutput(final JobContext context, final int jobId) { if (jobId == 0) return;//from ww w .j a va2s. c o m Connection con = null; FileSystem fileSystem = null; FSDataInputStream in = null; FSDataOutputStream out = null; try { fileSystem = outputPath.getFileSystem(context.getConfiguration()); con = DriverManager.getConnection(url, user, password); PreparedStatement ps = null; try { ps = con.prepareStatement("INSERT INTO boa_output (id, length) VALUES (" + jobId + ", 0)"); ps.executeUpdate(); } catch (final Exception e) { } finally { try { if (ps != null) ps.close(); } catch (final Exception e) { e.printStackTrace(); } } fileSystem.mkdirs(new Path("/boa", new Path("" + jobId))); out = fileSystem.create(new Path("/boa", new Path("" + jobId, new Path("output.txt")))); int partNum = 0; final byte[] b = new byte[64 * 1024 * 1024]; long length = 0; boolean hasWebResult = false; while (true) { final Path path = new Path(outputPath, "part-r-" + String.format("%05d", partNum++)); if (!fileSystem.exists(path)) break; if (in != null) try { in.close(); } catch (final Exception e) { e.printStackTrace(); } in = fileSystem.open(path); int numBytes = 0; while ((numBytes = in.read(b)) > 0) { if (!hasWebResult) { hasWebResult = true; try { ps = con.prepareStatement("UPDATE boa_output SET web_result=? WHERE id=" + jobId); int webSize = 64 * 1024 - 1; ps.setString(1, new String(b, 0, numBytes < webSize ? numBytes : webSize)); ps.executeUpdate(); } finally { try { if (ps != null) ps.close(); } catch (final Exception e) { e.printStackTrace(); } } } out.write(b, 0, numBytes); length += numBytes; this.context.progress(); } } try { ps = con.prepareStatement("UPDATE boa_output SET length=? WHERE id=" + jobId); ps.setLong(1, length); ps.executeUpdate(); } finally { try { if (ps != null) ps.close(); } catch (final Exception e) { e.printStackTrace(); } } } catch (final Exception e) { e.printStackTrace(); } finally { try { if (con != null) con.close(); } catch (final Exception e) { e.printStackTrace(); } try { if (in != null) in.close(); } catch (final Exception e) { e.printStackTrace(); } try { if (out != null) out.close(); } catch (final Exception e) { e.printStackTrace(); } try { if (fileSystem != null) fileSystem.close(); } catch (final Exception e) { e.printStackTrace(); } } }
From source file:boostingPL.MR.AdaBoostPLTestMapper.java
License:Open Source License
private void output2HDFS(Context context) throws Exception { int jobID = context.getJobID().getId(); int taskID = context.getTaskAttemptID().getTaskID().getId(); String outputFloder = context.getConfiguration().get("BoostingPL.outputFolder"); Path path = new Path(outputFloder + "/result_" + jobID + "_m_" + taskID); FileSystem hdfs = FileSystem.get(context.getConfiguration()); FSDataOutputStream outputStream = hdfs.create(path); String result = eval.toSummaryString(); outputStream.write(result.getBytes()); result = eval.toClassDetailsString(); outputStream.write(result.getBytes()); result = eval.toMatrixString();//from w ww . ja va 2s . c om outputStream.write(result.getBytes()); result = "-----------------------------------------------------------"; outputStream.write(result.getBytes()); outputStream.close(); }
From source file:boostingPL.MR.AdaBoostPLTestReducer.java
License:Open Source License
private void output2HDFS(Context context) throws Exception { int jobID = context.getJobID().getId(); int taskID = context.getTaskAttemptID().getTaskID().getId(); String outputFloder = context.getConfiguration().get("BoostingPL.outputFolder"); Path path = new Path(outputFloder + "/result_" + jobID + "_r_" + taskID); FileSystem hdfs = FileSystem.get(context.getConfiguration()); FSDataOutputStream outputStream = hdfs.create(path); String result = eval.toSummaryString(); outputStream.write(result.getBytes()); result = eval.toClassDetailsString(); outputStream.write(result.getBytes()); result = eval.toMatrixString();//from ww w . jav a 2 s . com outputStream.write(result.getBytes()); result = "-----------------------------------------------------------"; outputStream.write(result.getBytes()); outputStream.close(); }
From source file:cascading.tap.hadoop.ZipInputFormatTest.java
License:Open Source License
public void testSplits() throws Exception { JobConf job = new JobConf(); FileSystem currentFs = FileSystem.get(job); Path file = new Path(workDir, "test.zip"); Reporter reporter = Reporter.NULL;/*w w w .j a va2 s. com*/ int seed = new Random().nextInt(); LOG.info("seed = " + seed); Random random = new Random(seed); FileInputFormat.setInputPaths(job, file); for (int entries = 1; entries < MAX_ENTRIES; entries += random.nextInt(MAX_ENTRIES / 10) + 1) { ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); ZipOutputStream zos = new ZipOutputStream(byteArrayOutputStream); long length = 0; LOG.debug("creating; zip file with entries = " + entries); // for each entry in the zip file for (int entryCounter = 0; entryCounter < entries; entryCounter++) { // construct zip entries splitting MAX_LENGTH between entries long entryLength = MAX_LENGTH / entries; ZipEntry zipEntry = new ZipEntry("/entry" + entryCounter + ".txt"); zipEntry.setMethod(ZipEntry.DEFLATED); zos.putNextEntry(zipEntry); for (length = entryCounter * entryLength; length < (entryCounter + 1) * entryLength; length++) { zos.write(Long.toString(length).getBytes()); zos.write("\n".getBytes()); } zos.flush(); zos.closeEntry(); } zos.flush(); zos.close(); currentFs.delete(file, true); OutputStream outputStream = currentFs.create(file); byteArrayOutputStream.writeTo(outputStream); outputStream.close(); ZipInputFormat format = new ZipInputFormat(); format.configure(job); LongWritable key = new LongWritable(); Text value = new Text(); InputSplit[] splits = format.getSplits(job, 100); BitSet bits = new BitSet((int) length); for (int j = 0; j < splits.length; j++) { LOG.debug("split[" + j + "]= " + splits[j]); RecordReader<LongWritable, Text> reader = format.getRecordReader(splits[j], job, reporter); try { int count = 0; while (reader.next(key, value)) { int v = Integer.parseInt(value.toString()); LOG.debug("read " + v); if (bits.get(v)) LOG.warn("conflict with " + v + " in split " + j + " at position " + reader.getPos()); assertFalse("key in multiple partitions.", bits.get(v)); bits.set(v); count++; } LOG.debug("splits[" + j + "]=" + splits[j] + " count=" + count); } finally { reader.close(); } } assertEquals("some keys in no partition.", length, bits.cardinality()); } }
From source file:cc.solr.lucene.store.hdfs.HdfsFileWriter.java
License:Apache License
public HdfsFileWriter(FileSystem fileSystem, Path path) throws IOException { _outputStream = fileSystem.create(path); seek(0); }
From source file:cienciaCelularMR.ZipFileWriter.java
public void setup(Configuration conf) { FileSystem fs; try {//from ww w . jav a 2 s . co m fs = FileSystem.get(conf); Path path = new Path(zipFileName); os = fs.create(path); rootDir = "/data"; } catch (IOException e) { e.printStackTrace(); } log.info("CREADO " + zipFileName); }