List of usage examples for java.nio.channels FileChannel size
public abstract long size() throws IOException;
From source file:org.chromium.APKPackager.java
private void copyFile(File src, File dest) throws FileNotFoundException, IOException { FileInputStream istream = new FileInputStream(src); FileOutputStream ostream = new FileOutputStream(dest); FileChannel input = istream.getChannel(); FileChannel output = ostream.getChannel(); try {//w w w . j a v a2 s . c o m input.transferTo(0, input.size(), output); } finally { istream.close(); ostream.close(); input.close(); output.close(); } }
From source file:org.forgerock.openidm.script.javascript.JavaScriptFactory.java
private Script initializeScript(String name, File source, boolean sharedScope) throws ScriptException { initDebugListener();/*from w w w .j a va 2s. c o m*/ if (debugInitialised) { try { FileChannel inChannel = new FileInputStream(source).getChannel(); FileChannel outChannel = new FileOutputStream(getTargetFile(name)).getChannel(); FileLock outLock = outChannel.lock(); FileLock inLock = inChannel.lock(0, inChannel.size(), true); inChannel.transferTo(0, inChannel.size(), outChannel); outLock.release(); inLock.release(); inChannel.close(); outChannel.close(); } catch (IOException e) { logger.warn("JavaScript source was not updated for {}", name, e); } } return new JavaScript(name, source, sharedScope); }
From source file:com.linkedin.pinot.core.segment.creator.impl.inv.OffHeapBitmapInvertedIndexCreator.java
@Override public void seal() throws IOException { FileOutputStream fos = null;/*from w ww . j a v a 2 s. c om*/ FileInputStream fisOffsets = null; FileInputStream fisBitmaps = null; final DataOutputStream bitmapsOut; final DataOutputStream offsetsOut; String tempOffsetsFile = invertedIndexFile + ".offsets"; String tempBitmapsFile = invertedIndexFile + ".binary"; try { // build the posting list constructPostingLists(); // we need two separate streams, one to write the offsets and another to write the serialized // bitmap data. We need two because we dont the serialized length of each bitmap without // constructing. offsetsOut = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(tempOffsetsFile))); bitmapsOut = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(tempBitmapsFile))); // write out offsets of bitmaps. The information can be used to access a certain bitmap // directly. // Totally (invertedIndex.length+1) offsets will be written out; the last offset is used to // calculate the length of // the last bitmap, which might be needed when accessing bitmaps randomly. // If a bitmap's offset is k, then k bytes need to be skipped to reach the bitmap. int startOffset = 4 * (cardinality + 1); offsetsOut.writeInt(startOffset);// The first bitmap's offset MutableRoaringBitmap bitmap = new MutableRoaringBitmap(); for (int i = 0; i < cardinality; i++) { bitmap.clear(); int length = postingListLengths.get(i); for (int j = 0; j < length; j++) { int bufferOffset = postingListStartOffsets.get(i) + j; int value = postingListBuffer.get(bufferOffset); bitmap.add(value); } // serialize bitmap to bitmapsOut stream bitmap.serialize(bitmapsOut); startOffset += bitmap.serializedSizeInBytes(); // write offset offsetsOut.writeInt(startOffset); } offsetsOut.close(); bitmapsOut.close(); // merge the two files by simply writing offsets data first and then bitmap serialized data fos = new FileOutputStream(invertedIndexFile); fisOffsets = new FileInputStream(tempOffsetsFile); fisBitmaps = new FileInputStream(tempBitmapsFile); FileChannel channelOffsets = fisOffsets.getChannel(); channelOffsets.transferTo(0, channelOffsets.size(), fos.getChannel()); FileChannel channelBitmaps = fisBitmaps.getChannel(); channelBitmaps.transferTo(0, channelBitmaps.size(), fos.getChannel()); LOGGER.debug("persisted bitmap inverted index for column : " + spec.getName() + " in " + invertedIndexFile.getAbsolutePath()); } catch (Exception e) { LOGGER.error("Exception while creating bitmap index for column:" + spec.getName(), e); } finally { IOUtils.closeQuietly(fos); IOUtils.closeQuietly(fisOffsets); IOUtils.closeQuietly(fisOffsets); IOUtils.closeQuietly(fos); IOUtils.closeQuietly(fos); // MMaputils handles the null checks for buffer MmapUtils.unloadByteBuffer(origValueBuffer); origValueBuffer = null; valueBuffer = null; if (origLengths != null) { MmapUtils.unloadByteBuffer(origLengths); origLengths = null; lengths = null; } MmapUtils.unloadByteBuffer(origPostingListBuffer); origPostingListBuffer = null; postingListBuffer = null; MmapUtils.unloadByteBuffer(origPostingListCurrentOffsets); origPostingListCurrentOffsets = null; postingListCurrentOffsets = null; MmapUtils.unloadByteBuffer(origPostingListLengths); origPostingListLengths = null; postingListLengths = null; MmapUtils.unloadByteBuffer(origPostingListStartOffsets); origPostingListStartOffsets = null; postingListStartOffsets = null; FileUtils.deleteQuietly(new File(tempOffsetsFile)); FileUtils.deleteQuietly(new File(tempBitmapsFile)); } }
From source file:org.apache.hadoop.hdfs.client.ShortCircuitReplica.java
MappedByteBuffer loadMmapInternal() {
try {// ww w. j a va2s.c o m
FileChannel channel = dataStream.getChannel();
MappedByteBuffer mmap = channel.map(MapMode.READ_ONLY, 0, Math.min(Integer.MAX_VALUE, channel.size()));
if (LOG.isTraceEnabled()) {
LOG.trace(this + ": created mmap of size " + channel.size());
}
return mmap;
} catch (IOException e) {
LOG.warn(this + ": mmap error", e);
return null;
} catch (RuntimeException e) {
LOG.warn(this + ": mmap error", e);
return null;
}
}
From source file:org.grouplens.lenskit.data.dao.packed.BinaryIndexTableTest.java
@Test public void testMultipleEntries() throws IOException { File file = folder.newFile(); FileChannel chan = new RandomAccessFile(file, "rw").getChannel(); BinaryIndexTableWriter w = BinaryIndexTableWriter.create(BinaryFormat.create(), chan, 3); w.writeEntry(42, new int[] { 0 }); w.writeEntry(49, new int[] { 1, 3 }); w.writeEntry(67, new int[] { 2, 4 }); MappedByteBuffer buf = chan.map(FileChannel.MapMode.READ_ONLY, 0, chan.size()); BinaryIndexTable tbl = BinaryIndexTable.fromBuffer(3, buf); assertThat(tbl.getKeys(), contains(42L, 49L, 67L)); assertThat(tbl.getEntry(42), contains(0)); assertThat(tbl.getEntry(49), contains(1, 3)); assertThat(tbl.getEntry(67), contains(2, 4)); assertThat(tbl.getEntry(-1), nullValue()); }
From source file:org.apache.hadoop.hdfs.TestCrcCorruption.java
/** * check if DFS can handle corrupted CRC blocks *///from w ww . j ava2 s . c om private void thistest(Configuration conf, DFSTestUtil util) throws Exception { MiniDFSCluster cluster = null; int numDataNodes = 2; short replFactor = 2; Random random = new Random(); try { cluster = new MiniDFSCluster.Builder(conf).numNameNodes(1).numDataNodes(numDataNodes).build(); cluster.waitActive(); FileSystem fs = cluster.getFileSystem(); util.createFiles(fs, "/srcdat", replFactor); util.waitReplication(fs, "/srcdat", (short) 2); // Now deliberately remove/truncate meta blocks from the first // directory of the first datanode. The complete absense of a meta // file disallows this Datanode to send data to another datanode. // However, a client is alowed access to this block. // File storageDir = MiniDFSCluster.getStorageDir(0, 1); String bpid = cluster.getNamesystem().getBlockPoolId(); File data_dir = MiniDFSCluster.getFinalizedDir(storageDir, bpid); assertTrue("data directory does not exist", data_dir.exists()); File[] blocks = data_dir.listFiles(); assertTrue("Blocks do not exist in data-dir", (blocks != null) && (blocks.length > 0)); int num = 0; for (int idx = 0; idx < blocks.length; idx++) { if (blocks[idx].getName().startsWith("blk_") && blocks[idx].getName().endsWith(".meta")) { num++; if (num % 3 == 0) { // // remove .meta file // LOG.info("Deliberately removing file " + blocks[idx].getName()); assertTrue("Cannot remove file.", blocks[idx].delete()); } else if (num % 3 == 1) { // // shorten .meta file // RandomAccessFile file = new RandomAccessFile(blocks[idx], "rw"); FileChannel channel = file.getChannel(); int newsize = random.nextInt((int) channel.size() / 2); LOG.info("Deliberately truncating file " + blocks[idx].getName() + " to size " + newsize + " bytes."); channel.truncate(newsize); file.close(); } else { // // corrupt a few bytes of the metafile // RandomAccessFile file = new RandomAccessFile(blocks[idx], "rw"); FileChannel channel = file.getChannel(); long position = 0; // // The very first time, corrupt the meta header at offset 0 // if (num != 2) { position = (long) random.nextInt((int) channel.size()); } int length = random.nextInt((int) (channel.size() - position + 1)); byte[] buffer = new byte[length]; random.nextBytes(buffer); channel.write(ByteBuffer.wrap(buffer), position); LOG.info("Deliberately corrupting file " + blocks[idx].getName() + " at offset " + position + " length " + length); file.close(); } } } // // Now deliberately corrupt all meta blocks from the second // directory of the first datanode // storageDir = MiniDFSCluster.getStorageDir(0, 1); data_dir = MiniDFSCluster.getFinalizedDir(storageDir, bpid); assertTrue("data directory does not exist", data_dir.exists()); blocks = data_dir.listFiles(); assertTrue("Blocks do not exist in data-dir", (blocks != null) && (blocks.length > 0)); int count = 0; File previous = null; for (int idx = 0; idx < blocks.length; idx++) { if (blocks[idx].getName().startsWith("blk_") && blocks[idx].getName().endsWith(".meta")) { // // Move the previous metafile into the current one. // count++; if (count % 2 == 0) { LOG.info("Deliberately insertimg bad crc into files " + blocks[idx].getName() + " " + previous.getName()); assertTrue("Cannot remove file.", blocks[idx].delete()); assertTrue("Cannot corrupt meta file.", previous.renameTo(blocks[idx])); assertTrue("Cannot recreate empty meta file.", previous.createNewFile()); previous = null; } else { previous = blocks[idx]; } } } // // Only one replica is possibly corrupted. The other replica should still // be good. Verify. // assertTrue("Corrupted replicas not handled properly.", util.checkFiles(fs, "/srcdat")); LOG.info("All File still have a valid replica"); // // set replication factor back to 1. This causes only one replica of // of each block to remain in HDFS. The check is to make sure that // the corrupted replica generated above is the one that gets deleted. // This test is currently disabled until HADOOP-1557 is solved. // util.setReplication(fs, "/srcdat", (short) 1); //util.waitReplication(fs, "/srcdat", (short)1); //LOG.info("All Files done with removing replicas"); //assertTrue("Excess replicas deleted. Corrupted replicas found.", // util.checkFiles(fs, "/srcdat")); LOG.info("The excess-corrupted-replica test is disabled " + " pending HADOOP-1557"); util.cleanup(fs, "/srcdat"); } finally { if (cluster != null) { cluster.shutdown(); } } }
From source file:MyFormApp.java
void pdfToimage(File filename) throws FileNotFoundException, IOException { //?pdf ? // TODO Auto-generated method stub File pdfFile = new File(filename.toString()); // pdf RandomAccessFile raf = new RandomAccessFile(pdfFile, "r"); FileChannel channel = raf.getChannel(); ByteBuffer buf = channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size()); PDFFile pdf = new PDFFile(buf); int i = 0;/*w ww . j a v a 2 s . co m*/ String fileNameWithOutExt = FilenameUtils.removeExtension(filename.getName()); Rectangle rect = new Rectangle(0, 0, (int) pdf.getPage(i).getBBox().getWidth(), // (int) pdf.getPage(i).getBBox().getHeight()); BufferedImage bufferedImage = new BufferedImage(100, 100, BufferedImage.TYPE_INT_RGB); Image image = pdf.getPage(i).getImage(rect.width, rect.height, // width & height rect, // clip rect null, // null for the ImageObserver true, // fill background with white true // block until drawing is done ); Graphics2D bufImageGraphics = bufferedImage.createGraphics(); bufImageGraphics.drawImage(image.getScaledInstance(100, 100, Image.SCALE_AREA_AVERAGING), 0, 0, null); ImageIO.write(bufferedImage, "PNG", new File(PATH + fileNameWithOutExt + ".png")); //? }
From source file:com.homesnap.webserver.AbstractRestApi.java
protected void copyFileUsingFileChannels(File source, File dest) throws IOException { FileChannel inputChannel = null; FileChannel outputChannel = null; try {/*from ww w.j a v a 2 s . com*/ inputChannel = new FileInputStream(source).getChannel(); outputChannel = new FileOutputStream(dest).getChannel(); outputChannel.transferFrom(inputChannel, 0, inputChannel.size()); } finally { inputChannel.close(); outputChannel.close(); } }
From source file:pyromaniac.IO.MMFastqImporter.java
/** * Helper function for init(). Scans this.fastq file for sequence starts and records their position. * Multiple MappedByteBuffers are used to handle large files. * @throws Exceptions relating to file reading and decoding. *//*from w w w .j a v a 2s .c om*/ private void _initFile() throws Exception { FileInputStream tempStream = new FileInputStream(new File(this.fastqFile)); FileChannel fcSeq = tempStream.getChannel(); this.seqSizeLong = fcSeq.size(); this.recordStarts = new ArrayList<Pair<Integer, Long>>(); int state = -1; for (long startPosition = 0L; startPosition < this.seqSizeLong; startPosition += HALF_GIGA) { MappedByteBuffer recordBuffer = fcSeq.map(FileChannel.MapMode.READ_ONLY, startPosition, Math.min(this.seqSizeLong - startPosition, HALF_GIGA)); this.recordBuffers.add(recordBuffer); int sbf_pos = this.recordBuffers.size() - 1; int maxBuffer = 2048; int bufferSize = (recordBuffer.capacity() > maxBuffer) ? maxBuffer : recordBuffer.capacity(); recordBuffer.limit(bufferSize); recordBuffer.position(0); while (recordBuffer.position() != recordBuffer.capacity()) { int prevPos = recordBuffer.position(); CharBuffer result = decoder.decode(recordBuffer); recordBuffer.position(prevPos); for (int i = 0; i < result.capacity(); i++) { char curr = result.charAt(i); int posInFile = prevPos + i; //I see a fastq header, I am either at beginning of file, or last saw the quality line... if (curr == BEGINNING_FASTQ_SEQ && (state == -1 || state == 4)) { this.recordStarts.add(new Pair<Integer, Long>(sbf_pos, new Long(posInFile))); state = 1; } else if (curr == BEGINNING_FASTQ_QUAL && (state == 1)) { state = 2; } else if ((curr == '\n' || curr == '\r') & state == 2) { state = 3; } else if ((curr == '\n' || curr == '\r') & state == 3) { state = 4; } } int newPos = recordBuffer.limit(); if (recordBuffer.limit() + bufferSize > recordBuffer.capacity()) recordBuffer.limit(recordBuffer.capacity()); else recordBuffer.limit(recordBuffer.limit() + bufferSize); recordBuffer.position(newPos); } recordBuffer.rewind(); } }
From source file:net.ytbolg.mcxa.ImportOldMc.java
void fileChannelCopy(File s, File t) { FileInputStream fi = null;//from w w w. jav a2s. c om FileOutputStream fo = null; FileChannel in = null; FileChannel out = null; try { fi = new FileInputStream(s); fo = new FileOutputStream(t); in = fi.getChannel();//? out = fo.getChannel();//? in.transferTo(0, in.size(), out);//?in???out? } catch (IOException e) { e.printStackTrace(); } finally { try { fi.close(); in.close(); fo.close(); out.close(); } catch (IOException e) { e.printStackTrace(); } } }