Java tutorial
/** * Copyright 2014 NetApp Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package org.apache.hadoop.fs.nfs.stream; import java.io.IOException; import java.io.OutputStream; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.nfs.NFSv3FileSystemStore; import org.apache.hadoop.fs.nfs.StreamStatistics; import org.apache.hadoop.nfs.nfs3.FileHandle; import org.apache.hadoop.nfs.nfs3.Nfs3FileAttributes; import org.apache.hadoop.oncrpc.security.Credentials; public class NFSBufferedOutputStream extends OutputStream { final FileHandle handle; final Credentials credentials; final Path path; final String pathString; final StreamStatistics statistics; final NFSv3FileSystemStore store; final AtomicBoolean closed; final int blockSizeBits; final ExecutorService executors; final List<Future<Write>> ongoing; long fileOffset; StreamBlock currentBlock; private static final int MAX_WRITEBACK_POOL_SIZE = 256; private static final int DEFAULT_WRITEBACK_POOL_SIZE = 4; static final AtomicInteger streamId; public final static Log LOG = LogFactory.getLog(NFSBufferedOutputStream.class); static { streamId = new AtomicInteger(1); } public NFSBufferedOutputStream(Configuration configuration, FileHandle handle, Path path, NFSv3FileSystemStore store, Credentials credentials, boolean append) throws IOException { this.handle = handle; this.credentials = credentials; this.path = path; this.pathString = path.toUri().getPath(); this.statistics = new StreamStatistics(NFSBufferedInputStream.class + pathString, streamId.getAndIncrement(), false); this.store = store; this.blockSizeBits = store.getWriteSizeBits(); this.currentBlock = null; this.closed = new AtomicBoolean(false); assert (blockSizeBits >= 0 && blockSizeBits <= 22); // Create the task queues executors = new ThreadPoolExecutor(DEFAULT_WRITEBACK_POOL_SIZE, MAX_WRITEBACK_POOL_SIZE, 5, TimeUnit.SECONDS, new LinkedBlockingDeque<Runnable>(1024), new ThreadPoolExecutor.CallerRunsPolicy()); ongoing = new LinkedList<>(); // Set file offset to 0 or file length if (append) { Nfs3FileAttributes attributes = store.getFileAttributes(handle, credentials); if (attributes != null) { fileOffset = attributes.getSize(); LOG.info("Appending to file so starting at offset = " + fileOffset); } else { throw new IOException("Could not get file length"); } } else { fileOffset = 0L; } } @Override public void write(int b) throws IOException { byte buffer[] = new byte[1]; buffer[0] = (byte) b; _write(buffer, 0, 1); } @Override public void write(byte[] data) throws IOException { _write(data, 0, data.length); } @Override public void write(byte[] data, int offset, int length) throws IOException { long startTime = System.currentTimeMillis(); try { _write(data, offset, length); } finally { statistics.incrementBytesWritten(length); statistics.incrementWriteOps(1); statistics.incrementTimeWritten((System.currentTimeMillis() - startTime)); } } private synchronized void _write(byte[] data, int offset, int length) throws IOException { int lengthToWrite = Math.min(data.length, length); int blockSize = (int) (1 << blockSizeBits); long loBlockId = (long) (fileOffset >> blockSizeBits); long hiBlockId = (long) ((fileOffset + lengthToWrite - 1) >> blockSizeBits); int loOffset = (int) (fileOffset - (loBlockId << blockSizeBits)); int hiOffset = (int) ((fileOffset + lengthToWrite - 1) - (hiBlockId << blockSizeBits)); if (closed.get() == true) { LOG.warn("Trying to write to a closed stream. Check your code."); } // All the data is in one block, so it's easy to handle if (loBlockId == hiBlockId) { StreamBlock block = getBlock(loBlockId); assert (block != null); int bytesWritten = block.writeToBlock(data, offset, loOffset, lengthToWrite); assert (bytesWritten == lengthToWrite); fileOffset += bytesWritten; } // The data is in multiple blocks, so we need do much more work else { int totalBytesWritten = offset; for (long blk = loBlockId; blk <= hiBlockId; blk++) { StreamBlock block = getBlock(blk); assert (block != null); // We write from loOffset in the starting block if (blk == loBlockId) { int bytesWritten = block.writeToBlock(data, offset, loOffset, blockSize - loOffset); assert (bytesWritten == (blockSize - loOffset)); totalBytesWritten += bytesWritten; fileOffset += bytesWritten; } // We write up to hiOffset in the ending block else if (blk == hiBlockId) { int bytesWritten = block.writeToBlock(data, totalBytesWritten, 0, hiOffset + 1); assert (bytesWritten == (hiOffset + 1)); totalBytesWritten += bytesWritten; fileOffset += bytesWritten; } // Middle blocks are written fully else { int bytesWritten = block.writeToBlock(data, totalBytesWritten, 0, blockSize); assert (bytesWritten == blockSize); totalBytesWritten += bytesWritten; fileOffset += bytesWritten; } } } } @Override public synchronized void flush() throws IOException { try { if (closed.get() == true) { LOG.warn("Flushing a closed stream. Check your code."); } // Write back the current block if (currentBlock != null) { flushBlock(currentBlock); } // Commit all outstanding changes Commit commit = new Commit(this, store, handle, credentials, 0L, 0); Future<Commit> future = executors.submit(commit); while (true) { try { future.get(); break; } catch (InterruptedException exception) { LOG.info("Got interrupted while waiting for COMMIT to finish, trying again"); continue; } } } catch (Exception exception) { throw new IOException("Could not flush stream"); } } @Override public synchronized void close() throws IOException { boolean first = true; long start = System.currentTimeMillis(); if (closed.get() == true) { first = false; LOG.warn("Closing an already closed output stream"); } closed.set(true); // Shutdown the thread pool if (first) { flush(); executors.shutdown(); try { executors.awaitTermination(60, TimeUnit.SECONDS); } catch (InterruptedException exception) { // Ignore } } LOG.info(statistics); super.close(); LOG.info("OutputStream shutdown took " + (System.currentTimeMillis() - start) + " ms"); } private StreamBlock getBlock(long blockId) throws IOException { if (currentBlock != null && blockId == currentBlock.getBlockId()) { return currentBlock; } else { if (currentBlock != null) { flushBlock(currentBlock); } currentBlock = new StreamBlock(blockSizeBits); currentBlock.setBlockId(blockId); return currentBlock; } } private void checkOngoing() throws IOException { if (ongoing.size() >= 64) { for (Iterator<Future<Write>> iter = ongoing.iterator(); iter.hasNext();) { Future<Write> f = iter.next(); if (!f.isDone()) { try { f.get(); iter.remove(); } catch (InterruptedException interrupted) { // Ignore } catch (ExecutionException execution) { throw new IOException("Write back call failed", execution); } } } } } private void flushBlock(StreamBlock block) throws IOException { // Check ongoing checkOngoing(); // Submit new task Write call = new Write(store, handle, credentials, statistics, block.getBlockId(), currentBlock); Future<Write> future = executors.submit(call); ongoing.add(future); } }