Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hdfs.server.datanode; import java.io.BufferedInputStream; import java.io.DataInputStream; import java.io.FileDescriptor; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.channels.FileChannel; import org.apache.commons.logging.Log; import org.apache.hadoop.fs.FSDataNodeReadProfilingData; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.FSConstants; import org.apache.hadoop.hdfs.server.datanode.BlockSender.InputStreamFactory; import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.util.DataChecksum; /** * Base class for reading data for blocks from local disks and * fill in packet buffer. * * @author sdong * */ abstract class DatanodeBlockReader implements java.io.Closeable { static final Log LOG = DataNode.LOG; static final Log ClientTraceLog = DataNode.ClientTraceLog; protected int namespaceId; protected Block block; protected boolean isFinalized; protected FSDataNodeReadProfilingData dnData; protected boolean ignoreChecksum; protected boolean verifyChecksum; // if true, check is verified while reading protected boolean corruptChecksumOk; // if need to verify checksum protected DataChecksum checksum; // checksum stream protected int bytesPerChecksum; // chunk size protected int checksumSize; // checksum size DatanodeBlockReader(int namespaceId, Block block, boolean isFinalized, boolean ignoreChecksum, boolean verifyChecksum, boolean corruptChecksumOk) { this.namespaceId = namespaceId; this.block = block; this.isFinalized = isFinalized; this.ignoreChecksum = ignoreChecksum; this.verifyChecksum = verifyChecksum; this.corruptChecksumOk = corruptChecksumOk; } /** * Read metadata (file header) from disk and generate checksum * metadata of the block. * * @param blockLength * @return * @throws IOException */ public abstract DataChecksum getChecksumToSend(long blockLength) throws IOException; public abstract void fadviseStream(int advise, long offset, long len) throws IOException; /** * Initialize input file stream(s) of local files for the block. * * @param offset * @param blockLength * @throws IOException */ public abstract void initialize(long offset, long blockLength) throws IOException; /** * Pre-processing to determine whether it is possible to directly stream * data from data blocks to output stream. It should only worked for * the separate checksum file case. For other cases, a false should be * directly returned. * @return * @throws IOException */ public abstract boolean prepareTransferTo() throws IOException; /** * From current location, stream from current position to output stream * or packet buffer, based on the case. * * The expected format for the output packet buffer: * * checksumOff-->+---------------------+ * |Checksum for Chunk 1 | * +---------------------+ * |Checksum for Chunk 2 | * +---------------------+ * | . | * | . | * | . | * +---------------------+ * |Checksum for Chunk N | * +---------------------+ * | | * | | * | | * | Data | * | | * | | * | | * +---------------------+ * * @param out output stream to clients, only used in direct * transfer mode. In that case, buf is not used. * @param buf packet buffer for the data * @param offset starting block offset. * @param checksumOff starting checksum offset. * @param numChunks how many chunks to read * @param len how many bytes to read * @throws IOException */ public abstract void sendChunks(OutputStream out, byte[] buf, long offset, int checksumOff, int numChunks, int len, BlockCrcUpdater crcUpdater, int packetVersion) throws IOException; public abstract int getPreferredPacketVersion(); /** * @return number of bytes per data chunk for checksum */ int getBytesPerChecksum() { return bytesPerChecksum; } /** * @return number of bytes per checksum for a data chunk */ int getChecksumSize() { return checksumSize; } /** * Populate bytes per checksum and checksum size information from * local checksum object. * * @param blockLength */ protected void getChecksumInfo(long blockLength) { /* * If bytesPerChecksum is very large, then the metadata file is mostly * corrupted. For now just truncate bytesPerchecksum to blockLength. */ bytesPerChecksum = checksum.getBytesPerChecksum(); if (bytesPerChecksum > 10 * 1024 * 1024 && bytesPerChecksum > blockLength) { checksum = DataChecksum.newDataChecksum(checksum.getChecksumType(), Math.max((int) blockLength, 10 * 1024 * 1024)); bytesPerChecksum = checksum.getBytesPerChecksum(); } checksumSize = checksum.getChecksumSize(); } static class BlockInputStreamFactory implements BlockWithChecksumFileReader.InputStreamWithChecksumFactory { private final int namespaceId; private final Block block; private final ReplicaToRead replica; private final DataNode datanode; private final FSDatasetInterface data; private final boolean ignoreChecksum; private final boolean verifyChecksum; private final boolean corruptChecksumOk; protected DataInputStream metadataIn = null; BlockInputStreamFactory(int namespaceId, Block block, ReplicaToRead replica, DataNode datanode, FSDatasetInterface data, boolean ignoreChecksum, boolean verifyChecksum, boolean corruptChecksumOk) { this.namespaceId = namespaceId; this.block = block; this.replica = replica; this.datanode = datanode; this.data = data; this.ignoreChecksum = ignoreChecksum; this.verifyChecksum = verifyChecksum; this.corruptChecksumOk = corruptChecksumOk; } @Override public InputStream createStream(long offset) throws IOException { return replica.getBlockInputStream(datanode, offset); } @Override public DataInputStream getChecksumStream() throws IOException { if (replica == null || replica.isInlineChecksum()) { throw new IOException("Getting checksum stream for inline checksum stream is not supported"); } if (metadataIn != null) { return metadataIn; } if (!ignoreChecksum) { try { metadataIn = new DataInputStream(new BufferedInputStream( BlockWithChecksumFileReader.getMetaDataInputStream(data, namespaceId, block), FSConstants.BUFFER_SIZE)); } catch (IOException e) { if (!corruptChecksumOk || BlockWithChecksumFileReader.metaFileExists(data, namespaceId, block)) { throw e; } metadataIn = null; } } return metadataIn; } public DatanodeBlockReader getBlockReader() throws IOException { if (replica.isInlineChecksum()) { int checksumType = replica.getChecksumType(); int bytesPerChecksum = replica.getBytesPerChecksum(); return new BlockInlineChecksumReader(this.namespaceId, this.block, replica.isFinalized(), this.ignoreChecksum, this.verifyChecksum, this.corruptChecksumOk, this, checksumType, bytesPerChecksum); } else { return new BlockWithChecksumFileReader(this.namespaceId, this.block, replica.isFinalized(), this.ignoreChecksum, this.verifyChecksum, this.corruptChecksumOk, this); } } public FSDatasetInterface getDataset() { return this.data; } @Override public BlockDataFile.Reader getBlockDataFileReader() throws IOException { return replica.getBlockDataFile().getReader(datanode); } } public void enableReadProfiling(FSDataNodeReadProfilingData dnData) { this.dnData = dnData; } }