Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hdfs; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InterruptedIOException; import java.nio.ByteBuffer; import java.nio.channels.ClosedChannelException; import java.util.EnumSet; import java.util.concurrent.atomic.AtomicReference; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.crypto.CryptoProtocolVersion; import org.apache.hadoop.fs.CanSetDropBehind; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSOutputSummer; import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.StreamCapabilities; import org.apache.hadoop.fs.Syncable; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag; import org.apache.hadoop.hdfs.client.impl.DfsClientConf; import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException; import org.apache.hadoop.hdfs.protocol.QuotaByStorageTypeExceededException; import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException; import org.apache.hadoop.hdfs.protocol.UnresolvedPathException; import org.apache.hadoop.hdfs.protocol.datatransfer.PacketHeader; import org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.hdfs.server.datanode.CachingStrategy; import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException; import org.apache.hadoop.hdfs.server.namenode.RetryStartFileException; import org.apache.hadoop.hdfs.server.namenode.SafeModeException; import org.apache.hadoop.hdfs.util.ByteArrayManager; import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.io.MultipleIOException; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.DataChecksum.Type; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; import org.apache.htrace.core.TraceScope; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; /**************************************************************** * DFSOutputStream creates files from a stream of bytes. * * The client application writes data that is cached internally by * this stream. Data is broken up into packets, each packet is * typically 64K in size. A packet comprises of chunks. Each chunk * is typically 512 bytes and has an associated checksum with it. * * When a client application fills up the currentPacket, it is * enqueued into the dataQueue of DataStreamer. DataStreamer is a * thread that picks up packets from the dataQueue and sends it to * the first datanode in the pipeline. * ****************************************************************/ @InterfaceAudience.Private public class DFSOutputStream extends FSOutputSummer implements Syncable, CanSetDropBehind, StreamCapabilities { static final Logger LOG = LoggerFactory.getLogger(DFSOutputStream.class); /** * Number of times to retry creating a file when there are transient * errors (typically related to encryption zones and KeyProvider operations). */ @VisibleForTesting static final int CREATE_RETRY_COUNT = 10; @VisibleForTesting static CryptoProtocolVersion[] SUPPORTED_CRYPTO_VERSIONS = CryptoProtocolVersion.supported(); protected final DFSClient dfsClient; protected final ByteArrayManager byteArrayManager; // closed is accessed by different threads under different locks. protected volatile boolean closed = false; protected final String src; protected final long fileId; protected final long blockSize; protected final int bytesPerChecksum; protected DFSPacket currentPacket = null; protected DataStreamer streamer; protected int packetSize = 0; // write packet size, not including the header. protected int chunksPerPacket = 0; protected long lastFlushOffset = 0; // offset when flush was invoked private long initialFileSize = 0; // at time of file open private final short blockReplication; // replication factor of file protected boolean shouldSyncBlock = false; // force blocks to disk upon close private final EnumSet<AddBlockFlag> addBlockFlags; protected final AtomicReference<CachingStrategy> cachingStrategy; private FileEncryptionInfo fileEncryptionInfo; private int writePacketSize; /** Use {@link ByteArrayManager} to create buffer for non-heartbeat packets.*/ protected DFSPacket createPacket(int packetSize, int chunksPerPkt, long offsetInBlock, long seqno, boolean lastPacketInBlock) throws InterruptedIOException { final byte[] buf; final int bufferSize = PacketHeader.PKT_MAX_HEADER_LEN + packetSize; try { buf = byteArrayManager.newByteArray(bufferSize); } catch (InterruptedException ie) { final InterruptedIOException iioe = new InterruptedIOException("seqno=" + seqno); iioe.initCause(ie); throw iioe; } return new DFSPacket(buf, chunksPerPkt, offsetInBlock, seqno, getChecksumSize(), lastPacketInBlock); } @Override protected void checkClosed() throws IOException { if (isClosed()) { getStreamer().getLastException().throwException4Close(); } } // // returns the list of targets, if any, that is being currently used. // @VisibleForTesting public synchronized DatanodeInfo[] getPipeline() { if (getStreamer().streamerClosed()) { return null; } DatanodeInfo[] currentNodes = getStreamer().getNodes(); if (currentNodes == null) { return null; } DatanodeInfo[] value = new DatanodeInfo[currentNodes.length]; System.arraycopy(currentNodes, 0, value, 0, currentNodes.length); return value; } /** * @return the object for computing checksum. * The type is NULL if checksum is not computed. */ private static DataChecksum getChecksum4Compute(DataChecksum checksum, HdfsFileStatus stat) { if (DataStreamer.isLazyPersist(stat) && stat.getReplication() == 1) { // do not compute checksum for writing to single replica to memory return DataChecksum.newDataChecksum(Type.NULL, checksum.getBytesPerChecksum()); } return checksum; } private DFSOutputStream(DFSClient dfsClient, String src, EnumSet<CreateFlag> flag, Progressable progress, HdfsFileStatus stat, DataChecksum checksum) { super(getChecksum4Compute(checksum, stat)); this.dfsClient = dfsClient; this.src = src; this.fileId = stat.getFileId(); this.blockSize = stat.getBlockSize(); this.blockReplication = stat.getReplication(); this.fileEncryptionInfo = stat.getFileEncryptionInfo(); this.cachingStrategy = new AtomicReference<>(dfsClient.getDefaultWriteCachingStrategy()); this.addBlockFlags = EnumSet.noneOf(AddBlockFlag.class); if (flag.contains(CreateFlag.NO_LOCAL_WRITE)) { this.addBlockFlags.add(AddBlockFlag.NO_LOCAL_WRITE); } if (flag.contains(CreateFlag.IGNORE_CLIENT_LOCALITY)) { this.addBlockFlags.add(AddBlockFlag.IGNORE_CLIENT_LOCALITY); } if (progress != null) { DFSClient.LOG.debug("Set non-null progress callback on DFSOutputStream " + "{}", src); } initWritePacketSize(); this.bytesPerChecksum = checksum.getBytesPerChecksum(); if (bytesPerChecksum <= 0) { throw new HadoopIllegalArgumentException( "Invalid value: bytesPerChecksum = " + bytesPerChecksum + " <= 0"); } if (blockSize % bytesPerChecksum != 0) { throw new HadoopIllegalArgumentException( "Invalid values: " + HdfsClientConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY + " (=" + bytesPerChecksum + ") must divide block size (=" + blockSize + ")."); } this.byteArrayManager = dfsClient.getClientContext().getByteArrayManager(); } /** * Ensures the configured writePacketSize never exceeds * PacketReceiver.MAX_PACKET_SIZE. */ private void initWritePacketSize() { writePacketSize = dfsClient.getConf().getWritePacketSize(); if (writePacketSize > PacketReceiver.MAX_PACKET_SIZE) { LOG.warn("Configured write packet exceeds {} bytes as max," + " using {} bytes.", PacketReceiver.MAX_PACKET_SIZE, PacketReceiver.MAX_PACKET_SIZE); writePacketSize = PacketReceiver.MAX_PACKET_SIZE; } } /** Construct a new output stream for creating a file. */ protected DFSOutputStream(DFSClient dfsClient, String src, HdfsFileStatus stat, EnumSet<CreateFlag> flag, Progressable progress, DataChecksum checksum, String[] favoredNodes, boolean createStreamer) { this(dfsClient, src, flag, progress, stat, checksum); this.shouldSyncBlock = flag.contains(CreateFlag.SYNC_BLOCK); computePacketChunkSize(dfsClient.getConf().getWritePacketSize(), bytesPerChecksum); if (createStreamer) { streamer = new DataStreamer(stat, null, dfsClient, src, progress, checksum, cachingStrategy, byteArrayManager, favoredNodes, addBlockFlags); } } static DFSOutputStream newStreamForCreate(DFSClient dfsClient, String src, FsPermission masked, EnumSet<CreateFlag> flag, boolean createParent, short replication, long blockSize, Progressable progress, DataChecksum checksum, String[] favoredNodes, String ecPolicyName) throws IOException { try (TraceScope ignored = dfsClient.newPathTraceScope("newStreamForCreate", src)) { HdfsFileStatus stat = null; // Retry the create if we get a RetryStartFileException up to a maximum // number of times boolean shouldRetry = true; int retryCount = CREATE_RETRY_COUNT; while (shouldRetry) { shouldRetry = false; try { stat = dfsClient.namenode.create(src, masked, dfsClient.clientName, new EnumSetWritable<>(flag), createParent, replication, blockSize, SUPPORTED_CRYPTO_VERSIONS, ecPolicyName); break; } catch (RemoteException re) { IOException e = re.unwrapRemoteException(AccessControlException.class, DSQuotaExceededException.class, QuotaByStorageTypeExceededException.class, FileAlreadyExistsException.class, FileNotFoundException.class, ParentNotDirectoryException.class, NSQuotaExceededException.class, RetryStartFileException.class, SafeModeException.class, UnresolvedPathException.class, SnapshotAccessControlException.class, UnknownCryptoProtocolVersionException.class); if (e instanceof RetryStartFileException) { if (retryCount > 0) { shouldRetry = true; retryCount--; } else { throw new IOException("Too many retries because of encryption" + " zone operations", e); } } else { throw e; } } } Preconditions.checkNotNull(stat, "HdfsFileStatus should not be null!"); final DFSOutputStream out; if (stat.getErasureCodingPolicy() != null) { out = new DFSStripedOutputStream(dfsClient, src, stat, flag, progress, checksum, favoredNodes); } else { out = new DFSOutputStream(dfsClient, src, stat, flag, progress, checksum, favoredNodes, true); } out.start(); return out; } } /** Construct a new output stream for append. */ private DFSOutputStream(DFSClient dfsClient, String src, EnumSet<CreateFlag> flags, Progressable progress, LocatedBlock lastBlock, HdfsFileStatus stat, DataChecksum checksum, String[] favoredNodes) throws IOException { this(dfsClient, src, flags, progress, stat, checksum); initialFileSize = stat.getLen(); // length of file when opened this.shouldSyncBlock = flags.contains(CreateFlag.SYNC_BLOCK); boolean toNewBlock = flags.contains(CreateFlag.NEW_BLOCK); this.fileEncryptionInfo = stat.getFileEncryptionInfo(); // The last partial block of the file has to be filled. if (!toNewBlock && lastBlock != null) { // indicate that we are appending to an existing block streamer = new DataStreamer(lastBlock, stat, dfsClient, src, progress, checksum, cachingStrategy, byteArrayManager); getStreamer().setBytesCurBlock(lastBlock.getBlockSize()); adjustPacketChunkSize(stat); getStreamer().setPipelineInConstruction(lastBlock); } else { computePacketChunkSize(dfsClient.getConf().getWritePacketSize(), bytesPerChecksum); streamer = new DataStreamer(stat, lastBlock != null ? lastBlock.getBlock() : null, dfsClient, src, progress, checksum, cachingStrategy, byteArrayManager, favoredNodes, addBlockFlags); } } private void adjustPacketChunkSize(HdfsFileStatus stat) throws IOException { long usedInLastBlock = stat.getLen() % blockSize; int freeInLastBlock = (int) (blockSize - usedInLastBlock); // calculate the amount of free space in the pre-existing // last crc chunk int usedInCksum = (int) (stat.getLen() % bytesPerChecksum); int freeInCksum = bytesPerChecksum - usedInCksum; // if there is space in the last block, then we have to // append to that block if (freeInLastBlock == blockSize) { throw new IOException("The last block for file " + src + " is full."); } if (usedInCksum > 0 && freeInCksum > 0) { // if there is space in the last partial chunk, then // setup in such a way that the next packet will have only // one chunk that fills up the partial chunk. // computePacketChunkSize(0, freeInCksum); setChecksumBufSize(freeInCksum); getStreamer().setAppendChunk(true); } else { // if the remaining space in the block is smaller than // that expected size of of a packet, then create // smaller size packet. // computePacketChunkSize(Math.min(dfsClient.getConf().getWritePacketSize(), freeInLastBlock), bytesPerChecksum); } } static DFSOutputStream newStreamForAppend(DFSClient dfsClient, String src, EnumSet<CreateFlag> flags, Progressable progress, LocatedBlock lastBlock, HdfsFileStatus stat, DataChecksum checksum, String[] favoredNodes) throws IOException { if (stat.getErasureCodingPolicy() != null) { throw new IOException("Not support appending to a striping layout file yet."); } try (TraceScope ignored = dfsClient.newPathTraceScope("newStreamForAppend", src)) { final DFSOutputStream out = new DFSOutputStream(dfsClient, src, flags, progress, lastBlock, stat, checksum, favoredNodes); out.start(); return out; } } protected void computePacketChunkSize(int psize, int csize) { final int bodySize = psize - PacketHeader.PKT_MAX_HEADER_LEN; final int chunkSize = csize + getChecksumSize(); chunksPerPacket = Math.max(bodySize / chunkSize, 1); packetSize = chunkSize * chunksPerPacket; DFSClient.LOG.debug("computePacketChunkSize: src={}, chunkSize={}, " + "chunksPerPacket={}, packetSize={}", src, chunkSize, chunksPerPacket, packetSize); } protected TraceScope createWriteTraceScope() { return dfsClient.newPathTraceScope("DFSOutputStream#write", src); } // @see FSOutputSummer#writeChunk() @Override protected synchronized void writeChunk(byte[] b, int offset, int len, byte[] checksum, int ckoff, int cklen) throws IOException { writeChunkPrepare(len, ckoff, cklen); currentPacket.writeChecksum(checksum, ckoff, cklen); currentPacket.writeData(b, offset, len); currentPacket.incNumChunks(); getStreamer().incBytesCurBlock(len); // If packet is full, enqueue it for transmission if (currentPacket.getNumChunks() == currentPacket.getMaxChunks() || getStreamer().getBytesCurBlock() == blockSize) { enqueueCurrentPacketFull(); } } /* write the data chunk in <code>buffer</code> staring at * <code>buffer.position</code> with * a length of <code>len > 0</code>, and its checksum */ protected synchronized void writeChunk(ByteBuffer buffer, int len, byte[] checksum, int ckoff, int cklen) throws IOException { writeChunkPrepare(len, ckoff, cklen); currentPacket.writeChecksum(checksum, ckoff, cklen); currentPacket.writeData(buffer, len); currentPacket.incNumChunks(); getStreamer().incBytesCurBlock(len); // If packet is full, enqueue it for transmission if (currentPacket.getNumChunks() == currentPacket.getMaxChunks() || getStreamer().getBytesCurBlock() == blockSize) { enqueueCurrentPacketFull(); } } private synchronized void writeChunkPrepare(int buflen, int ckoff, int cklen) throws IOException { dfsClient.checkOpen(); checkClosed(); if (buflen > bytesPerChecksum) { throw new IOException("writeChunk() buffer size is " + buflen + " is larger than supported bytesPerChecksum " + bytesPerChecksum); } if (cklen != 0 && cklen != getChecksumSize()) { throw new IOException("writeChunk() checksum size is supposed to be " + getChecksumSize() + " but found to be " + cklen); } if (currentPacket == null) { currentPacket = createPacket(packetSize, chunksPerPacket, getStreamer().getBytesCurBlock(), getStreamer().getAndIncCurrentSeqno(), false); DFSClient.LOG.debug( "WriteChunk allocating new packet seqno={}," + " src={}, packetSize={}, chunksPerPacket={}, bytesCurBlock={}", currentPacket.getSeqno(), src, packetSize, chunksPerPacket, getStreamer().getBytesCurBlock() + ", " + this); } } void enqueueCurrentPacket() throws IOException { getStreamer().waitAndQueuePacket(currentPacket); currentPacket = null; } synchronized void enqueueCurrentPacketFull() throws IOException { LOG.debug("enqueue full {}, src={}, bytesCurBlock={}, blockSize={}," + " appendChunk={}, {}", currentPacket, src, getStreamer().getBytesCurBlock(), blockSize, getStreamer().getAppendChunk(), getStreamer()); enqueueCurrentPacket(); adjustChunkBoundary(); endBlock(); } /** create an empty packet to mark the end of the block. */ void setCurrentPacketToEmpty() throws InterruptedIOException { currentPacket = createPacket(0, 0, getStreamer().getBytesCurBlock(), getStreamer().getAndIncCurrentSeqno(), true); currentPacket.setSyncBlock(shouldSyncBlock); } /** * If the reopened file did not end at chunk boundary and the above * write filled up its partial chunk. Tell the summer to generate full * crc chunks from now on. */ protected void adjustChunkBoundary() { if (getStreamer().getAppendChunk() && getStreamer().getBytesCurBlock() % bytesPerChecksum == 0) { getStreamer().setAppendChunk(false); resetChecksumBufSize(); } if (!getStreamer().getAppendChunk()) { final int psize = (int) Math.min(blockSize - getStreamer().getBytesCurBlock(), writePacketSize); computePacketChunkSize(psize, bytesPerChecksum); } } /** * Used in test only. */ @VisibleForTesting void setAppendChunk(final boolean appendChunk) { getStreamer().setAppendChunk(appendChunk); } /** * Used in test only. */ @VisibleForTesting void setBytesCurBlock(final long bytesCurBlock) { getStreamer().setBytesCurBlock(bytesCurBlock); } /** * if encountering a block boundary, send an empty packet to * indicate the end of block and reset bytesCurBlock. * * @throws IOException */ void endBlock() throws IOException { if (getStreamer().getBytesCurBlock() == blockSize) { setCurrentPacketToEmpty(); enqueueCurrentPacket(); getStreamer().setBytesCurBlock(0); lastFlushOffset = 0; } } @Override public boolean hasCapability(String capability) { switch (StringUtils.toLowerCase(capability)) { case StreamCapabilities.HSYNC: case StreamCapabilities.HFLUSH: return true; default: return false; } } /** * Flushes out to all replicas of the block. The data is in the buffers * of the DNs but not necessarily in the DN's OS buffers. * * It is a synchronous operation. When it returns, * it guarantees that flushed data become visible to new readers. * It is not guaranteed that data has been flushed to * persistent store on the datanode. * Block allocations are persisted on namenode. */ @Override public void hflush() throws IOException { try (TraceScope ignored = dfsClient.newPathTraceScope("hflush", src)) { flushOrSync(false, EnumSet.noneOf(SyncFlag.class)); } } @Override public void hsync() throws IOException { try (TraceScope ignored = dfsClient.newPathTraceScope("hsync", src)) { flushOrSync(true, EnumSet.noneOf(SyncFlag.class)); } } /** * The expected semantics is all data have flushed out to all replicas * and all replicas have done posix fsync equivalent - ie the OS has * flushed it to the disk device (but the disk may have it in its cache). * * Note that only the current block is flushed to the disk device. * To guarantee durable sync across block boundaries the stream should * be created with {@link CreateFlag#SYNC_BLOCK}. * * @param syncFlags * Indicate the semantic of the sync. Currently used to specify * whether or not to update the block length in NameNode. */ public void hsync(EnumSet<SyncFlag> syncFlags) throws IOException { try (TraceScope ignored = dfsClient.newPathTraceScope("hsync", src)) { flushOrSync(true, syncFlags); } } /** * Flush/Sync buffered data to DataNodes. * * @param isSync * Whether or not to require all replicas to flush data to the disk * device * @param syncFlags * Indicate extra detailed semantic of the flush/sync. Currently * mainly used to specify whether or not to update the file length in * the NameNode * @throws IOException */ private void flushOrSync(boolean isSync, EnumSet<SyncFlag> syncFlags) throws IOException { dfsClient.checkOpen(); checkClosed(); try { long toWaitFor; long lastBlockLength = -1L; boolean updateLength = syncFlags.contains(SyncFlag.UPDATE_LENGTH); boolean endBlock = syncFlags.contains(SyncFlag.END_BLOCK); synchronized (this) { // flush checksum buffer, but keep checksum buffer intact if we do not // need to end the current block int numKept = flushBuffer(!endBlock, true); // bytesCurBlock potentially incremented if there was buffered data DFSClient.LOG.debug( "DFSClient flush(): bytesCurBlock={}, " + "lastFlushOffset={}, createNewBlock={}", getStreamer().getBytesCurBlock(), lastFlushOffset, endBlock); // Flush only if we haven't already flushed till this offset. if (lastFlushOffset != getStreamer().getBytesCurBlock()) { assert getStreamer().getBytesCurBlock() > lastFlushOffset; // record the valid offset of this flush lastFlushOffset = getStreamer().getBytesCurBlock(); if (isSync && currentPacket == null && !endBlock) { // Nothing to send right now, // but sync was requested. // Send an empty packet if we do not end the block right now currentPacket = createPacket(packetSize, chunksPerPacket, getStreamer().getBytesCurBlock(), getStreamer().getAndIncCurrentSeqno(), false); } } else { if (isSync && getStreamer().getBytesCurBlock() > 0 && !endBlock) { // Nothing to send right now, // and the block was partially written, // and sync was requested. // So send an empty sync packet if we do not end the block right // now currentPacket = createPacket(packetSize, chunksPerPacket, getStreamer().getBytesCurBlock(), getStreamer().getAndIncCurrentSeqno(), false); } else if (currentPacket != null) { // just discard the current packet since it is already been sent. currentPacket.releaseBuffer(byteArrayManager); currentPacket = null; } } if (currentPacket != null) { currentPacket.setSyncBlock(isSync); enqueueCurrentPacket(); } if (endBlock && getStreamer().getBytesCurBlock() > 0) { // Need to end the current block, thus send an empty packet to // indicate this is the end of the block and reset bytesCurBlock currentPacket = createPacket(0, 0, getStreamer().getBytesCurBlock(), getStreamer().getAndIncCurrentSeqno(), true); currentPacket.setSyncBlock(shouldSyncBlock || isSync); enqueueCurrentPacket(); getStreamer().setBytesCurBlock(0); lastFlushOffset = 0; } else { // Restore state of stream. Record the last flush offset // of the last full chunk that was flushed. getStreamer().setBytesCurBlock(getStreamer().getBytesCurBlock() - numKept); } toWaitFor = getStreamer().getLastQueuedSeqno(); } // end synchronized getStreamer().waitForAckedSeqno(toWaitFor); // update the block length first time irrespective of flag if (updateLength || getStreamer().getPersistBlocks().get()) { synchronized (this) { if (!getStreamer().streamerClosed() && getStreamer().getBlock() != null) { lastBlockLength = getStreamer().getBlock().getNumBytes(); } } } // If 1) any new blocks were allocated since the last flush, or 2) to // update length in NN is required, then persist block locations on // namenode. if (getStreamer().getPersistBlocks().getAndSet(false) || updateLength) { try { dfsClient.namenode.fsync(src, fileId, dfsClient.clientName, lastBlockLength); } catch (IOException ioe) { DFSClient.LOG.warn("Unable to persist blocks in hflush for " + src, ioe); // If we got an error here, it might be because some other thread // called close before our hflush completed. In that case, we should // throw an exception that the stream is closed. checkClosed(); // If we aren't closed but failed to sync, we should expose that to // the caller. throw ioe; } } synchronized (this) { if (!getStreamer().streamerClosed()) { getStreamer().setHflush(); } } } catch (InterruptedIOException interrupt) { // This kind of error doesn't mean that the stream itself is broken - just // the flushing thread got interrupted. So, we shouldn't close down the // writer, but instead just propagate the error throw interrupt; } catch (IOException e) { DFSClient.LOG.warn("Error while syncing", e); synchronized (this) { if (!isClosed()) { getStreamer().getLastException().set(e); closeThreads(true); } } throw e; } } /** * @deprecated use {@link HdfsDataOutputStream#getCurrentBlockReplication()}. */ @Deprecated public synchronized int getNumCurrentReplicas() throws IOException { return getCurrentBlockReplication(); } /** * Note that this is not a public API; * use {@link HdfsDataOutputStream#getCurrentBlockReplication()} instead. * * @return the number of valid replicas of the current block */ public synchronized int getCurrentBlockReplication() throws IOException { dfsClient.checkOpen(); checkClosed(); if (getStreamer().streamerClosed()) { return blockReplication; // no pipeline, return repl factor of file } DatanodeInfo[] currentNodes = getStreamer().getNodes(); if (currentNodes == null) { return blockReplication; // no pipeline, return repl factor of file } return currentNodes.length; } /** * Waits till all existing data is flushed and confirmations * received from datanodes. */ protected void flushInternal() throws IOException { long toWaitFor = flushInternalWithoutWaitingAck(); getStreamer().waitForAckedSeqno(toWaitFor); } protected synchronized void start() { getStreamer().start(); } /** * Aborts this output stream and releases any system * resources associated with this stream. */ void abort() throws IOException { final MultipleIOException.Builder b = new MultipleIOException.Builder(); synchronized (this) { if (isClosed()) { return; } getStreamer().getLastException().set(new IOException( "Lease timeout of " + (dfsClient.getConf().getHdfsTimeout() / 1000) + " seconds expired.")); try { closeThreads(true); } catch (IOException e) { b.add(e); } } final IOException ioe = b.build(); if (ioe != null) { throw ioe; } } boolean isClosed() { return closed || getStreamer().streamerClosed(); } void setClosed() { closed = true; dfsClient.endFileLease(fileId); getStreamer().release(); } // shutdown datastreamer and responseprocessor threads. // interrupt datastreamer if force is true protected void closeThreads(boolean force) throws IOException { try { getStreamer().close(force); getStreamer().join(); getStreamer().closeSocket(); } catch (InterruptedException e) { throw new IOException("Failed to shutdown streamer"); } finally { getStreamer().setSocketToNull(); setClosed(); } } /** * Closes this output stream and releases any system * resources associated with this stream. */ @Override public void close() throws IOException { final MultipleIOException.Builder b = new MultipleIOException.Builder(); synchronized (this) { try (TraceScope ignored = dfsClient.newPathTraceScope("DFSOutputStream#close", src)) { closeImpl(); } catch (IOException e) { b.add(e); } } final IOException ioe = b.build(); if (ioe != null) { throw ioe; } } protected synchronized void closeImpl() throws IOException { if (isClosed()) { LOG.debug("Closing an already closed stream. [Stream:{}, streamer:{}]", closed, getStreamer().streamerClosed()); try { getStreamer().getLastException().check(true); } catch (IOException ioe) { cleanupAndRethrowIOException(ioe); } finally { if (!closed) { // If stream is not closed but streamer closed, clean up the stream. // Most importantly, end the file lease. closeThreads(true); } } return; } try { flushBuffer(); // flush from all upper layers if (currentPacket != null) { enqueueCurrentPacket(); } if (getStreamer().getBytesCurBlock() != 0) { setCurrentPacketToEmpty(); } try { flushInternal(); // flush all data to Datanodes } catch (IOException ioe) { cleanupAndRethrowIOException(ioe); } completeFile(); } catch (ClosedChannelException ignored) { } finally { // Failures may happen when flushing data. // Streamers may keep waiting for the new block information. // Thus need to force closing these threads. // Don't need to call setClosed() because closeThreads(true) // calls setClosed() in the finally block. closeThreads(true); } } private void completeFile() throws IOException { // get last block before destroying the streamer ExtendedBlock lastBlock = getStreamer().getBlock(); try (TraceScope ignored = dfsClient.getTracer().newScope("completeFile")) { completeFile(lastBlock); } } /** * Determines whether an IOException thrown needs extra cleanup on the stream. * Space quota exceptions will be thrown when getting new blocks, so the * open HDFS file need to be closed. * * @param ioe the IOException * @return whether the stream needs cleanup for the given IOException */ private boolean exceptionNeedsCleanup(IOException ioe) { return ioe instanceof DSQuotaExceededException || ioe instanceof QuotaByStorageTypeExceededException; } private void cleanupAndRethrowIOException(IOException ioe) throws IOException { if (exceptionNeedsCleanup(ioe)) { final MultipleIOException.Builder b = new MultipleIOException.Builder(); b.add(ioe); try { completeFile(); } catch (IOException e) { b.add(e); throw b.build(); } } throw ioe; } // should be called holding (this) lock since setTestFilename() may // be called during unit tests protected void completeFile(ExtendedBlock last) throws IOException { long localstart = Time.monotonicNow(); final DfsClientConf conf = dfsClient.getConf(); long sleeptime = conf.getBlockWriteLocateFollowingInitialDelayMs(); boolean fileComplete = false; int retries = conf.getNumBlockWriteLocateFollowingRetry(); while (!fileComplete) { fileComplete = dfsClient.namenode.complete(src, dfsClient.clientName, last, fileId); if (!fileComplete) { final int hdfsTimeout = conf.getHdfsTimeout(); if (!dfsClient.clientRunning || (hdfsTimeout > 0 && localstart + hdfsTimeout < Time.monotonicNow())) { String msg = "Unable to close file because dfsclient " + " was unable to contact the HDFS servers. clientRunning " + dfsClient.clientRunning + " hdfsTimeout " + hdfsTimeout; DFSClient.LOG.info(msg); throw new IOException(msg); } try { if (retries == 0) { throw new IOException("Unable to close file because the last block" + last + " does not have enough number of replicas."); } retries--; Thread.sleep(sleeptime); sleeptime *= 2; if (Time.monotonicNow() - localstart > 5000) { DFSClient.LOG.info("Could not complete " + src + " retrying..."); } } catch (InterruptedException ie) { DFSClient.LOG.warn("Caught exception ", ie); } } } } @VisibleForTesting public void setArtificialSlowdown(long period) { getStreamer().setArtificialSlowdown(period); } @VisibleForTesting public synchronized void setChunksPerPacket(int value) { chunksPerPacket = Math.min(chunksPerPacket, value); packetSize = (bytesPerChecksum + getChecksumSize()) * chunksPerPacket; } /** * Returns the size of a file as it was when this stream was opened */ public long getInitialLen() { return initialFileSize; } protected EnumSet<AddBlockFlag> getAddBlockFlags() { return addBlockFlags; } /** * @return the FileEncryptionInfo for this stream, or null if not encrypted. */ public FileEncryptionInfo getFileEncryptionInfo() { return fileEncryptionInfo; } /** * Returns the access token currently used by streamer, for testing only */ synchronized Token<BlockTokenIdentifier> getBlockToken() { return getStreamer().getBlockToken(); } protected long flushInternalWithoutWaitingAck() throws IOException { long toWaitFor; synchronized (this) { dfsClient.checkOpen(); checkClosed(); // // If there is data in the current buffer, send it across // getStreamer().queuePacket(currentPacket); currentPacket = null; toWaitFor = getStreamer().getLastQueuedSeqno(); } return toWaitFor; } @Override public void setDropBehind(Boolean dropBehind) throws IOException { CachingStrategy prevStrategy, nextStrategy; // CachingStrategy is immutable. So build a new CachingStrategy with the // modifications we want, and compare-and-swap it in. do { prevStrategy = this.cachingStrategy.get(); nextStrategy = new CachingStrategy.Builder(prevStrategy).setDropBehind(dropBehind).build(); } while (!this.cachingStrategy.compareAndSet(prevStrategy, nextStrategy)); } @VisibleForTesting ExtendedBlock getBlock() { return getStreamer().getBlock(); } @VisibleForTesting public long getFileId() { return fileId; } /** * Return the source of stream. */ String getSrc() { return src; } /** * Returns the data streamer object. */ protected DataStreamer getStreamer() { return streamer; } @Override public String toString() { return getClass().getSimpleName() + ":" + streamer; } static LocatedBlock addBlock(DatanodeInfo[] excludedNodes, DFSClient dfsClient, String src, ExtendedBlock prevBlock, long fileId, String[] favoredNodes, EnumSet<AddBlockFlag> allocFlags) throws IOException { final DfsClientConf conf = dfsClient.getConf(); int retries = conf.getNumBlockWriteLocateFollowingRetry(); long sleeptime = conf.getBlockWriteLocateFollowingInitialDelayMs(); long localstart = Time.monotonicNow(); while (true) { try { return dfsClient.namenode.addBlock(src, dfsClient.clientName, prevBlock, excludedNodes, fileId, favoredNodes, allocFlags); } catch (RemoteException e) { IOException ue = e.unwrapRemoteException(FileNotFoundException.class, AccessControlException.class, NSQuotaExceededException.class, DSQuotaExceededException.class, QuotaByStorageTypeExceededException.class, UnresolvedPathException.class); if (ue != e) { throw ue; // no need to retry these exceptions } if (NotReplicatedYetException.class.getName().equals(e.getClassName())) { if (retries == 0) { throw e; } else { --retries; LOG.info("Exception while adding a block", e); long elapsed = Time.monotonicNow() - localstart; if (elapsed > 5000) { LOG.info("Waiting for replication for " + (elapsed / 1000) + " seconds"); } try { LOG.warn("NotReplicatedYetException sleeping " + src + " retries left " + retries); Thread.sleep(sleeptime); sleeptime *= 2; } catch (InterruptedException ie) { LOG.warn("Caught exception", ie); } } } else { throw e; } } } } }