Java tutorial
/** * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hbase.regionserver; import java.io.DataInput; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.Map; import java.util.SortedSet; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HDFSBlocksDistribution; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; import org.apache.hadoop.hbase.io.hfile.BlockType; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileContext; import org.apache.hadoop.hbase.io.hfile.HFileScanner; import org.apache.hadoop.hbase.io.hfile.HFileWriterV2; import org.apache.hadoop.hbase.regionserver.compactions.Compactor; import org.apache.hadoop.hbase.util.BloomFilter; import org.apache.hadoop.hbase.util.BloomFilterFactory; import org.apache.hadoop.hbase.util.BloomFilterWriter; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ChecksumType; import org.apache.hadoop.hbase.util.Writables; import org.apache.hadoop.io.WritableUtils; import com.google.common.base.Function; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.Ordering; /** * A Store data file. Stores usually have one or more of these files. They * are produced by flushing the memstore to disk. To * create, instantiate a writer using {@link StoreFile.WriterBuilder} * and append data. Be sure to add any metadata before calling close on the * Writer (Use the appendMetadata convenience methods). On close, a StoreFile * is sitting in the Filesystem. To refer to it, create a StoreFile instance * passing filesystem and path. To read, call {@link #createReader()}. * <p>StoreFiles may also reference store files in another Store. * * The reason for this weird pattern where you use a different instance for the * writer and a reader is that we write once but read a lot more. */ @InterfaceAudience.LimitedPrivate("Coprocessor") public class StoreFile { static final Log LOG = LogFactory.getLog(StoreFile.class.getName()); // Keys for fileinfo values in HFile /** Max Sequence ID in FileInfo */ public static final byte[] MAX_SEQ_ID_KEY = Bytes.toBytes("MAX_SEQ_ID_KEY"); /** Major compaction flag in FileInfo */ public static final byte[] MAJOR_COMPACTION_KEY = Bytes.toBytes("MAJOR_COMPACTION_KEY"); /** Minor compaction flag in FileInfo */ public static final byte[] EXCLUDE_FROM_MINOR_COMPACTION_KEY = Bytes.toBytes("EXCLUDE_FROM_MINOR_COMPACTION"); /** Bloom filter Type in FileInfo */ public static final byte[] BLOOM_FILTER_TYPE_KEY = Bytes.toBytes("BLOOM_FILTER_TYPE"); /** Delete Family Count in FileInfo */ public static final byte[] DELETE_FAMILY_COUNT = Bytes.toBytes("DELETE_FAMILY_COUNT"); /** Last Bloom filter key in FileInfo */ private static final byte[] LAST_BLOOM_KEY = Bytes.toBytes("LAST_BLOOM_KEY"); /** Key for Timerange information in metadata*/ public static final byte[] TIMERANGE_KEY = Bytes.toBytes("TIMERANGE"); /** Key for timestamp of earliest-put in metadata*/ public static final byte[] EARLIEST_PUT_TS = Bytes.toBytes("EARLIEST_PUT_TS"); private final StoreFileInfo fileInfo; private final FileSystem fs; // Block cache configuration and reference. private final CacheConfig cacheConf; // Keys for metadata stored in backing HFile. // Set when we obtain a Reader. private long sequenceid = -1; // max of the MemstoreTS in the KV's in this store // Set when we obtain a Reader. private long maxMemstoreTS = -1; public long getMaxMemstoreTS() { return maxMemstoreTS; } public void setMaxMemstoreTS(long maxMemstoreTS) { this.maxMemstoreTS = maxMemstoreTS; } // If true, this file was product of a major compaction. Its then set // whenever you get a Reader. private AtomicBoolean majorCompaction = null; // If true, this file should not be included in minor compactions. // It's set whenever you get a Reader. private boolean excludeFromMinorCompaction = false; /** Meta key set when store file is a result of a bulk load */ public static final byte[] BULKLOAD_TASK_KEY = Bytes.toBytes("BULKLOAD_SOURCE_TASK"); public static final byte[] BULKLOAD_TIME_KEY = Bytes.toBytes("BULKLOAD_TIMESTAMP"); /** * Map of the metadata entries in the corresponding HFile */ private Map<byte[], byte[]> metadataMap; // StoreFile.Reader private volatile Reader reader; /** * Bloom filter type specified in column family configuration. Does not * necessarily correspond to the Bloom filter type present in the HFile. */ private final BloomType cfBloomType; // the last modification time stamp private long modificationTimeStamp = 0L; /** * Constructor, loads a reader and it's indices, etc. May allocate a * substantial amount of ram depending on the underlying files (10-20MB?). * * @param fs The current file system to use. * @param p The path of the file. * @param conf The current configuration. * @param cacheConf The cache configuration and block cache reference. * @param cfBloomType The bloom type to use for this store file as specified * by column family configuration. This may or may not be the same * as the Bloom filter type actually present in the HFile, because * column family configuration might change. If this is * {@link BloomType#NONE}, the existing Bloom filter is ignored. * @throws IOException When opening the reader fails. */ public StoreFile(final FileSystem fs, final Path p, final Configuration conf, final CacheConfig cacheConf, final BloomType cfBloomType) throws IOException { this(fs, new StoreFileInfo(conf, fs, p), conf, cacheConf, cfBloomType); } /** * Constructor, loads a reader and it's indices, etc. May allocate a * substantial amount of ram depending on the underlying files (10-20MB?). * * @param fs The current file system to use. * @param fileInfo The store file information. * @param conf The current configuration. * @param cacheConf The cache configuration and block cache reference. * @param cfBloomType The bloom type to use for this store file as specified * by column family configuration. This may or may not be the same * as the Bloom filter type actually present in the HFile, because * column family configuration might change. If this is * {@link BloomType#NONE}, the existing Bloom filter is ignored. * @throws IOException When opening the reader fails. */ public StoreFile(final FileSystem fs, final StoreFileInfo fileInfo, final Configuration conf, final CacheConfig cacheConf, final BloomType cfBloomType) throws IOException { this.fs = fs; this.fileInfo = fileInfo; this.cacheConf = cacheConf; if (BloomFilterFactory.isGeneralBloomEnabled(conf)) { this.cfBloomType = cfBloomType; } else { LOG.info("Ignoring bloom filter check for file " + this.getPath() + ": " + "cfBloomType=" + cfBloomType + " (disabled in config)"); this.cfBloomType = BloomType.NONE; } // cache the modification time stamp of this store file this.modificationTimeStamp = fileInfo.getModificationTime(); } /** * @return the StoreFile object associated to this StoreFile. * null if the StoreFile is not a reference. */ public StoreFileInfo getFileInfo() { return this.fileInfo; } /** * @return Path or null if this StoreFile was made with a Stream. */ public Path getPath() { return this.fileInfo.getPath(); } /** * @return Returns the qualified path of this StoreFile */ public Path getQualifiedPath() { return this.fileInfo.getPath().makeQualified(fs); } /** * @return True if this is a StoreFile Reference; call after {@link #open()} * else may get wrong answer. */ public boolean isReference() { return this.fileInfo.isReference(); } /** * @return True if this file was made by a major compaction. */ public boolean isMajorCompaction() { if (this.majorCompaction == null) { throw new NullPointerException("This has not been set yet"); } return this.majorCompaction.get(); } /** * @return True if this file should not be part of a minor compaction. */ public boolean excludeFromMinorCompaction() { return this.excludeFromMinorCompaction; } /** * @return This files maximum edit sequence id. */ public long getMaxSequenceId() { return this.sequenceid; } public long getModificationTimeStamp() { return modificationTimeStamp; } public byte[] getMetadataValue(byte[] key) { return metadataMap.get(key); } /** * Return the largest memstoreTS found across all storefiles in * the given list. Store files that were created by a mapreduce * bulk load are ignored, as they do not correspond to any specific * put operation, and thus do not have a memstoreTS associated with them. * @return 0 if no non-bulk-load files are provided or, this is Store that * does not yet have any store files. */ public static long getMaxMemstoreTSInList(Collection<StoreFile> sfs) { long max = 0; for (StoreFile sf : sfs) { if (!sf.isBulkLoadResult()) { max = Math.max(max, sf.getMaxMemstoreTS()); } } return max; } /** * Return the highest sequence ID found across all storefiles in * the given list. * @param sfs * @return 0 if no non-bulk-load files are provided or, this is Store that * does not yet have any store files. */ public static long getMaxSequenceIdInList(Collection<StoreFile> sfs) { long max = 0; for (StoreFile sf : sfs) { max = Math.max(max, sf.getMaxSequenceId()); } return max; } /** * @return true if this storefile was created by HFileOutputFormat * for a bulk load. */ boolean isBulkLoadResult() { return metadataMap.containsKey(BULKLOAD_TIME_KEY); } /** * Return the timestamp at which this bulk load file was generated. */ public long getBulkLoadTimestamp() { return Bytes.toLong(metadataMap.get(BULKLOAD_TIME_KEY)); } /** * @return the cached value of HDFS blocks distribution. The cached value is * calculated when store file is opened. */ public HDFSBlocksDistribution getHDFSBlockDistribution() { return this.fileInfo.getHDFSBlockDistribution(); } /** * Opens reader on this store file. Called by Constructor. * @return Reader for the store file. * @throws IOException * @see #closeReader(boolean) */ private Reader open() throws IOException { if (this.reader != null) { throw new IllegalAccessError("Already open"); } // Open the StoreFile.Reader this.reader = fileInfo.open(this.fs, this.cacheConf); // Load up indices and fileinfo. This also loads Bloom filter type. metadataMap = Collections.unmodifiableMap(this.reader.loadFileInfo()); // Read in our metadata. byte[] b = metadataMap.get(MAX_SEQ_ID_KEY); if (b != null) { // By convention, if halfhfile, top half has a sequence number > bottom // half. Thats why we add one in below. Its done for case the two halves // are ever merged back together --rare. Without it, on open of store, // since store files are distinguished by sequence id, the one half would // subsume the other. this.sequenceid = Bytes.toLong(b); if (fileInfo.isTopReference()) { this.sequenceid += 1; } } if (isBulkLoadResult()) { // generate the sequenceId from the fileName // fileName is of the form <randomName>_SeqId_<id-when-loaded>_ String fileName = this.getPath().getName(); int startPos = fileName.indexOf("SeqId_"); if (startPos != -1) { this.sequenceid = Long .parseLong(fileName.substring(startPos + 6, fileName.indexOf('_', startPos + 6))); // Handle reference files as done above. if (fileInfo.isTopReference()) { this.sequenceid += 1; } } } this.reader.setSequenceID(this.sequenceid); b = metadataMap.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY); if (b != null) { this.maxMemstoreTS = Bytes.toLong(b); } b = metadataMap.get(MAJOR_COMPACTION_KEY); if (b != null) { boolean mc = Bytes.toBoolean(b); if (this.majorCompaction == null) { this.majorCompaction = new AtomicBoolean(mc); } else { this.majorCompaction.set(mc); } } else { // Presume it is not major compacted if it doesn't explicity say so // HFileOutputFormat explicitly sets the major compacted key. this.majorCompaction = new AtomicBoolean(false); } b = metadataMap.get(EXCLUDE_FROM_MINOR_COMPACTION_KEY); this.excludeFromMinorCompaction = (b != null && Bytes.toBoolean(b)); BloomType hfileBloomType = reader.getBloomFilterType(); if (cfBloomType != BloomType.NONE) { reader.loadBloomfilter(BlockType.GENERAL_BLOOM_META); if (hfileBloomType != cfBloomType) { LOG.info("HFile Bloom filter type for " + reader.getHFileReader().getName() + ": " + hfileBloomType + ", but " + cfBloomType + " specified in column family " + "configuration"); } } else if (hfileBloomType != BloomType.NONE) { LOG.info("Bloom filter turned off by CF config for " + reader.getHFileReader().getName()); } // load delete family bloom filter reader.loadBloomfilter(BlockType.DELETE_FAMILY_BLOOM_META); try { byte[] timerangeBytes = metadataMap.get(TIMERANGE_KEY); if (timerangeBytes != null) { this.reader.timeRangeTracker = new TimeRangeTracker(); Writables.copyWritable(timerangeBytes, this.reader.timeRangeTracker); } } catch (IllegalArgumentException e) { LOG.error("Error reading timestamp range data from meta -- " + "proceeding without", e); this.reader.timeRangeTracker = null; } return this.reader; } /** * @return Reader for StoreFile. creates if necessary * @throws IOException */ public Reader createReader() throws IOException { if (this.reader == null) { try { this.reader = open(); } catch (IOException e) { try { this.closeReader(true); } catch (IOException ee) { } throw e; } } return this.reader; } /** * @return Current reader. Must call createReader first else returns null. * @see #createReader() */ public Reader getReader() { return this.reader; } /** * @param evictOnClose whether to evict blocks belonging to this file * @throws IOException */ public synchronized void closeReader(boolean evictOnClose) throws IOException { if (this.reader != null) { this.reader.close(evictOnClose); this.reader = null; } } /** * Delete this file * @throws IOException */ public void deleteReader() throws IOException { closeReader(true); this.fs.delete(getPath(), true); } @Override public String toString() { return this.fileInfo.toString(); } /** * @return a length description of this StoreFile, suitable for debug output */ public String toStringDetailed() { StringBuilder sb = new StringBuilder(); sb.append(this.getPath().toString()); sb.append(", isReference=").append(isReference()); sb.append(", isBulkLoadResult=").append(isBulkLoadResult()); if (isBulkLoadResult()) { sb.append(", bulkLoadTS=").append(getBulkLoadTimestamp()); } else { sb.append(", seqid=").append(getMaxSequenceId()); } sb.append(", majorCompaction=").append(isMajorCompaction()); return sb.toString(); } public static class WriterBuilder { private final Configuration conf; private final CacheConfig cacheConf; private final FileSystem fs; private KeyValue.KVComparator comparator = KeyValue.COMPARATOR; private BloomType bloomType = BloomType.NONE; private long maxKeyCount = 0; private Path dir; private Path filePath; private InetSocketAddress[] favoredNodes; private HFileContext fileContext; public WriterBuilder(Configuration conf, CacheConfig cacheConf, FileSystem fs) { this.conf = conf; this.cacheConf = cacheConf; this.fs = fs; } /** * Use either this method or {@link #withFilePath}, but not both. * @param dir Path to column family directory. The directory is created if * does not exist. The file is given a unique name within this * directory. * @return this (for chained invocation) */ public WriterBuilder withOutputDir(Path dir) { Preconditions.checkNotNull(dir); this.dir = dir; return this; } /** * Use either this method or {@link #withOutputDir}, but not both. * @param filePath the StoreFile path to write * @return this (for chained invocation) */ public WriterBuilder withFilePath(Path filePath) { Preconditions.checkNotNull(filePath); this.filePath = filePath; return this; } /** * @param favoredNodes an array of favored nodes or possibly null * @return this (for chained invocation) */ public WriterBuilder withFavoredNodes(InetSocketAddress[] favoredNodes) { this.favoredNodes = favoredNodes; return this; } public WriterBuilder withComparator(KeyValue.KVComparator comparator) { Preconditions.checkNotNull(comparator); this.comparator = comparator; return this; } public WriterBuilder withBloomType(BloomType bloomType) { Preconditions.checkNotNull(bloomType); this.bloomType = bloomType; return this; } /** * @param maxKeyCount estimated maximum number of keys we expect to add * @return this (for chained invocation) */ public WriterBuilder withMaxKeyCount(long maxKeyCount) { this.maxKeyCount = maxKeyCount; return this; } public WriterBuilder withFileContext(HFileContext fileContext) { this.fileContext = fileContext; return this; } /** * Create a store file writer. Client is responsible for closing file when * done. If metadata, add BEFORE closing using * {@link Writer#appendMetadata}. */ public Writer build() throws IOException { if ((dir == null ? 0 : 1) + (filePath == null ? 0 : 1) != 1) { throw new IllegalArgumentException("Either specify parent directory " + "or file path"); } if (dir == null) { dir = filePath.getParent(); } if (!fs.exists(dir)) { fs.mkdirs(dir); } if (filePath == null) { filePath = getUniqueFile(fs, dir); if (!BloomFilterFactory.isGeneralBloomEnabled(conf)) { bloomType = BloomType.NONE; } } if (comparator == null) { comparator = KeyValue.COMPARATOR; } return new Writer(fs, filePath, conf, cacheConf, comparator, bloomType, maxKeyCount, favoredNodes, fileContext); } } /** * @param fs * @param dir Directory to create file in. * @return random filename inside passed <code>dir</code> */ public static Path getUniqueFile(final FileSystem fs, final Path dir) throws IOException { if (!fs.getFileStatus(dir).isDirectory()) { throw new IOException("Expecting " + dir.toString() + " to be a directory"); } return new Path(dir, UUID.randomUUID().toString().replaceAll("-", "")); } public Long getMinimumTimestamp() { return (getReader().timeRangeTracker == null) ? null : getReader().timeRangeTracker.getMinimumTimestamp(); } /** * Gets the approximate mid-point of this file that is optimal for use in splitting it. * @param comparator Comparator used to compare KVs. * @return The split point row, or null if splitting is not possible, or reader is null. */ @SuppressWarnings("deprecation") byte[] getFileSplitPoint(KVComparator comparator) throws IOException { if (this.reader == null) { LOG.warn("Storefile " + this + " Reader is null; cannot get split point"); return null; } // Get first, last, and mid keys. Midkey is the key that starts block // in middle of hfile. Has column and timestamp. Need to return just // the row we want to split on as midkey. byte[] midkey = this.reader.midkey(); if (midkey != null) { KeyValue mk = KeyValue.createKeyValueFromKey(midkey, 0, midkey.length); byte[] fk = this.reader.getFirstKey(); KeyValue firstKey = KeyValue.createKeyValueFromKey(fk, 0, fk.length); byte[] lk = this.reader.getLastKey(); KeyValue lastKey = KeyValue.createKeyValueFromKey(lk, 0, lk.length); // if the midkey is the same as the first or last keys, we cannot (ever) split this region. if (comparator.compareRows(mk, firstKey) == 0 || comparator.compareRows(mk, lastKey) == 0) { if (LOG.isDebugEnabled()) { LOG.debug("cannot split because midkey is the same as first or last row"); } return null; } return mk.getRow(); } return null; } /** * A StoreFile writer. Use this to read/write HBase Store Files. It is package * local because it is an implementation detail of the HBase regionserver. */ public static class Writer implements Compactor.CellSink { private final BloomFilterWriter generalBloomFilterWriter; private final BloomFilterWriter deleteFamilyBloomFilterWriter; private final BloomType bloomType; private byte[] lastBloomKey; private int lastBloomKeyOffset, lastBloomKeyLen; private KVComparator kvComparator; private KeyValue lastKv = null; private long earliestPutTs = HConstants.LATEST_TIMESTAMP; private KeyValue lastDeleteFamilyKV = null; private long deleteFamilyCnt = 0; /** Checksum type */ protected ChecksumType checksumType; /** Bytes per Checksum */ protected int bytesPerChecksum; TimeRangeTracker timeRangeTracker = new TimeRangeTracker(); /* isTimeRangeTrackerSet keeps track if the timeRange has already been set * When flushing a memstore, we set TimeRange and use this variable to * indicate that it doesn't need to be calculated again while * appending KeyValues. * It is not set in cases of compactions when it is recalculated using only * the appended KeyValues*/ boolean isTimeRangeTrackerSet = false; protected HFile.Writer writer; /** * Creates an HFile.Writer that also write helpful meta data. * @param fs file system to write to * @param path file name to create * @param conf user configuration * @param comparator key comparator * @param bloomType bloom filter setting * @param maxKeys the expected maximum number of keys to be added. Was used * for Bloom filter size in {@link HFile} format version 1. * @param favoredNodes * @param fileContext - The HFile context * @throws IOException problem writing to FS */ private Writer(FileSystem fs, Path path, final Configuration conf, CacheConfig cacheConf, final KVComparator comparator, BloomType bloomType, long maxKeys, InetSocketAddress[] favoredNodes, HFileContext fileContext) throws IOException { writer = HFile.getWriterFactory(conf, cacheConf).withPath(fs, path).withComparator(comparator) .withFavoredNodes(favoredNodes).withFileContext(fileContext).create(); this.kvComparator = comparator; generalBloomFilterWriter = BloomFilterFactory.createGeneralBloomAtWrite(conf, cacheConf, bloomType, (int) Math.min(maxKeys, Integer.MAX_VALUE), writer); if (generalBloomFilterWriter != null) { this.bloomType = bloomType; if (LOG.isTraceEnabled()) LOG.trace("Bloom filter type for " + path + ": " + this.bloomType + ", " + generalBloomFilterWriter.getClass().getSimpleName()); } else { // Not using Bloom filters. this.bloomType = BloomType.NONE; } // initialize delete family Bloom filter when there is NO RowCol Bloom // filter if (this.bloomType != BloomType.ROWCOL) { this.deleteFamilyBloomFilterWriter = BloomFilterFactory.createDeleteBloomAtWrite(conf, cacheConf, (int) Math.min(maxKeys, Integer.MAX_VALUE), writer); } else { deleteFamilyBloomFilterWriter = null; } if (deleteFamilyBloomFilterWriter != null) { if (LOG.isTraceEnabled()) LOG.trace("Delete Family Bloom filter type for " + path + ": " + deleteFamilyBloomFilterWriter.getClass().getSimpleName()); } } /** * Writes meta data. * Call before {@link #close()} since its written as meta data to this file. * @param maxSequenceId Maximum sequence id. * @param majorCompaction True if this file is product of a major compaction * @throws IOException problem writing to FS */ public void appendMetadata(final long maxSequenceId, final boolean majorCompaction) throws IOException { writer.appendFileInfo(MAX_SEQ_ID_KEY, Bytes.toBytes(maxSequenceId)); writer.appendFileInfo(MAJOR_COMPACTION_KEY, Bytes.toBytes(majorCompaction)); appendTrackedTimestampsToMetadata(); } /** * Add TimestampRange and earliest put timestamp to Metadata */ public void appendTrackedTimestampsToMetadata() throws IOException { appendFileInfo(TIMERANGE_KEY, WritableUtils.toByteArray(timeRangeTracker)); appendFileInfo(EARLIEST_PUT_TS, Bytes.toBytes(earliestPutTs)); } /** * Set TimeRangeTracker * @param trt */ public void setTimeRangeTracker(final TimeRangeTracker trt) { this.timeRangeTracker = trt; isTimeRangeTrackerSet = true; } /** * Record the earlest Put timestamp. * * If the timeRangeTracker is not set, * update TimeRangeTracker to include the timestamp of this key * @param kv */ public void trackTimestamps(final KeyValue kv) { if (KeyValue.Type.Put.getCode() == kv.getTypeByte()) { earliestPutTs = Math.min(earliestPutTs, kv.getTimestamp()); } if (!isTimeRangeTrackerSet) { timeRangeTracker.includeTimestamp(kv); } } private void appendGeneralBloomfilter(final KeyValue kv) throws IOException { if (this.generalBloomFilterWriter != null) { // only add to the bloom filter on a new, unique key boolean newKey = true; if (this.lastKv != null) { switch (bloomType) { case ROW: newKey = !kvComparator.matchingRows(kv, lastKv); break; case ROWCOL: newKey = !kvComparator.matchingRowColumn(kv, lastKv); break; case NONE: newKey = false; break; default: throw new IOException( "Invalid Bloom filter type: " + bloomType + " (ROW or ROWCOL expected)"); } } if (newKey) { /* * http://2.bp.blogspot.com/_Cib_A77V54U/StZMrzaKufI/AAAAAAAAADo/ZhK7bGoJdMQ/s400/KeyValue.png * Key = RowLen + Row + FamilyLen + Column [Family + Qualifier] + TimeStamp * * 2 Types of Filtering: * 1. Row = Row * 2. RowCol = Row + Qualifier */ byte[] bloomKey; int bloomKeyOffset, bloomKeyLen; switch (bloomType) { case ROW: bloomKey = kv.getRowArray(); bloomKeyOffset = kv.getRowOffset(); bloomKeyLen = kv.getRowLength(); break; case ROWCOL: // merge(row, qualifier) // TODO: could save one buffer copy in case of compound Bloom // filters when this involves creating a KeyValue bloomKey = generalBloomFilterWriter.createBloomKey(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength()); bloomKeyOffset = 0; bloomKeyLen = bloomKey.length; break; default: throw new IOException( "Invalid Bloom filter type: " + bloomType + " (ROW or ROWCOL expected)"); } generalBloomFilterWriter.add(bloomKey, bloomKeyOffset, bloomKeyLen); if (lastBloomKey != null && generalBloomFilterWriter.getComparator().compareFlatKey(bloomKey, bloomKeyOffset, bloomKeyLen, lastBloomKey, lastBloomKeyOffset, lastBloomKeyLen) <= 0) { throw new IOException("Non-increasing Bloom keys: " + Bytes.toStringBinary(bloomKey, bloomKeyOffset, bloomKeyLen) + " after " + Bytes.toStringBinary(lastBloomKey, lastBloomKeyOffset, lastBloomKeyLen)); } lastBloomKey = bloomKey; lastBloomKeyOffset = bloomKeyOffset; lastBloomKeyLen = bloomKeyLen; this.lastKv = kv; } } } private void appendDeleteFamilyBloomFilter(final KeyValue kv) throws IOException { if (!kv.isDeleteFamily() && !kv.isDeleteFamilyVersion()) { return; } // increase the number of delete family in the store file deleteFamilyCnt++; if (null != this.deleteFamilyBloomFilterWriter) { boolean newKey = true; if (lastDeleteFamilyKV != null) { newKey = !kvComparator.matchingRows(kv, lastDeleteFamilyKV); } if (newKey) { this.deleteFamilyBloomFilterWriter.add(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength()); this.lastDeleteFamilyKV = kv; } } } public void append(final KeyValue kv) throws IOException { appendGeneralBloomfilter(kv); appendDeleteFamilyBloomFilter(kv); writer.append(kv); trackTimestamps(kv); } public Path getPath() { return this.writer.getPath(); } boolean hasGeneralBloom() { return this.generalBloomFilterWriter != null; } /** * For unit testing only. * * @return the Bloom filter used by this writer. */ BloomFilterWriter getGeneralBloomWriter() { return generalBloomFilterWriter; } private boolean closeBloomFilter(BloomFilterWriter bfw) throws IOException { boolean haveBloom = (bfw != null && bfw.getKeyCount() > 0); if (haveBloom) { bfw.compactBloom(); } return haveBloom; } private boolean closeGeneralBloomFilter() throws IOException { boolean hasGeneralBloom = closeBloomFilter(generalBloomFilterWriter); // add the general Bloom filter writer and append file info if (hasGeneralBloom) { writer.addGeneralBloomFilter(generalBloomFilterWriter); writer.appendFileInfo(BLOOM_FILTER_TYPE_KEY, Bytes.toBytes(bloomType.toString())); if (lastBloomKey != null) { writer.appendFileInfo(LAST_BLOOM_KEY, Arrays.copyOfRange(lastBloomKey, lastBloomKeyOffset, lastBloomKeyOffset + lastBloomKeyLen)); } } return hasGeneralBloom; } private boolean closeDeleteFamilyBloomFilter() throws IOException { boolean hasDeleteFamilyBloom = closeBloomFilter(deleteFamilyBloomFilterWriter); // add the delete family Bloom filter writer if (hasDeleteFamilyBloom) { writer.addDeleteFamilyBloomFilter(deleteFamilyBloomFilterWriter); } // append file info about the number of delete family kvs // even if there is no delete family Bloom. writer.appendFileInfo(DELETE_FAMILY_COUNT, Bytes.toBytes(this.deleteFamilyCnt)); return hasDeleteFamilyBloom; } public void close() throws IOException { boolean hasGeneralBloom = this.closeGeneralBloomFilter(); boolean hasDeleteFamilyBloom = this.closeDeleteFamilyBloomFilter(); writer.close(); // Log final Bloom filter statistics. This needs to be done after close() // because compound Bloom filters might be finalized as part of closing. if (StoreFile.LOG.isTraceEnabled()) { StoreFile.LOG.trace( (hasGeneralBloom ? "" : "NO ") + "General Bloom and " + (hasDeleteFamilyBloom ? "" : "NO ") + "DeleteFamily" + " was added to HFile " + getPath()); } } public void appendFileInfo(byte[] key, byte[] value) throws IOException { writer.appendFileInfo(key, value); } /** For use in testing, e.g. {@link org.apache.hadoop.hbase.regionserver.CreateRandomStoreFile} */ HFile.Writer getHFileWriter() { return writer; } } /** * Reader for a StoreFile. */ public static class Reader { static final Log LOG = LogFactory.getLog(Reader.class.getName()); protected BloomFilter generalBloomFilter = null; protected BloomFilter deleteFamilyBloomFilter = null; protected BloomType bloomFilterType; private final HFile.Reader reader; protected TimeRangeTracker timeRangeTracker = null; protected long sequenceID = -1; private byte[] lastBloomKey; private long deleteFamilyCnt = -1; public Reader(FileSystem fs, Path path, CacheConfig cacheConf, Configuration conf) throws IOException { reader = HFile.createReader(fs, path, cacheConf, conf); bloomFilterType = BloomType.NONE; } public Reader(FileSystem fs, Path path, FSDataInputStreamWrapper in, long size, CacheConfig cacheConf, Configuration conf) throws IOException { reader = HFile.createReader(fs, path, in, size, cacheConf, conf); bloomFilterType = BloomType.NONE; } /** * ONLY USE DEFAULT CONSTRUCTOR FOR UNIT TESTS */ Reader() { this.reader = null; } public KVComparator getComparator() { return reader.getComparator(); } /** * Get a scanner to scan over this StoreFile. Do not use * this overload if using this scanner for compactions. * * @param cacheBlocks should this scanner cache blocks? * @param pread use pread (for highly concurrent small readers) * @return a scanner */ public StoreFileScanner getStoreFileScanner(boolean cacheBlocks, boolean pread) { return getStoreFileScanner(cacheBlocks, pread, false, // 0 is passed as readpoint because this method is only used by test // where StoreFile is directly operated upon 0); } /** * Get a scanner to scan over this StoreFile. * * @param cacheBlocks should this scanner cache blocks? * @param pread use pread (for highly concurrent small readers) * @param isCompaction is scanner being used for compaction? * @return a scanner */ public StoreFileScanner getStoreFileScanner(boolean cacheBlocks, boolean pread, boolean isCompaction, long readPt) { return new StoreFileScanner(this, getScanner(cacheBlocks, pread, isCompaction), !isCompaction, reader.hasMVCCInfo(), readPt); } /** * Warning: Do not write further code which depends on this call. Instead * use getStoreFileScanner() which uses the StoreFileScanner class/interface * which is the preferred way to scan a store with higher level concepts. * * @param cacheBlocks should we cache the blocks? * @param pread use pread (for concurrent small readers) * @return the underlying HFileScanner */ @Deprecated public HFileScanner getScanner(boolean cacheBlocks, boolean pread) { return getScanner(cacheBlocks, pread, false); } /** * Warning: Do not write further code which depends on this call. Instead * use getStoreFileScanner() which uses the StoreFileScanner class/interface * which is the preferred way to scan a store with higher level concepts. * * @param cacheBlocks * should we cache the blocks? * @param pread * use pread (for concurrent small readers) * @param isCompaction * is scanner being used for compaction? * @return the underlying HFileScanner */ @Deprecated public HFileScanner getScanner(boolean cacheBlocks, boolean pread, boolean isCompaction) { return reader.getScanner(cacheBlocks, pread, isCompaction); } public void close(boolean evictOnClose) throws IOException { reader.close(evictOnClose); } /** * Check if this storeFile may contain keys within the TimeRange that * have not expired (i.e. not older than oldestUnexpiredTS). * @param scan the current scan * @param oldestUnexpiredTS the oldest timestamp that is not expired, as * determined by the column family's TTL * @return false if queried keys definitely don't exist in this StoreFile */ boolean passesTimerangeFilter(Scan scan, long oldestUnexpiredTS) { if (timeRangeTracker == null) { return true; } else { return timeRangeTracker.includesTimeRange(scan.getTimeRange()) && timeRangeTracker.getMaximumTimestamp() >= oldestUnexpiredTS; } } /** * Checks whether the given scan passes the Bloom filter (if present). Only * checks Bloom filters for single-row or single-row-column scans. Bloom * filter checking for multi-gets is implemented as part of the store * scanner system (see {@link StoreFileScanner#seekExactly}) and uses * the lower-level API {@link #passesGeneralBloomFilter(byte[], int, int, byte[], * int, int)}. * * @param scan the scan specification. Used to determine the row, and to * check whether this is a single-row ("get") scan. * @param columns the set of columns. Only used for row-column Bloom * filters. * @return true if the scan with the given column set passes the Bloom * filter, or if the Bloom filter is not applicable for the scan. * False if the Bloom filter is applicable and the scan fails it. */ boolean passesBloomFilter(Scan scan, final SortedSet<byte[]> columns) { // Multi-column non-get scans will use Bloom filters through the // lower-level API function that this function calls. if (!scan.isGetScan()) { return true; } byte[] row = scan.getStartRow(); switch (this.bloomFilterType) { case ROW: return passesGeneralBloomFilter(row, 0, row.length, null, 0, 0); case ROWCOL: if (columns != null && columns.size() == 1) { byte[] column = columns.first(); return passesGeneralBloomFilter(row, 0, row.length, column, 0, column.length); } // For multi-column queries the Bloom filter is checked from the // seekExact operation. return true; default: return true; } } public boolean passesDeleteFamilyBloomFilter(byte[] row, int rowOffset, int rowLen) { // Cache Bloom filter as a local variable in case it is set to null by // another thread on an IO error. BloomFilter bloomFilter = this.deleteFamilyBloomFilter; // Empty file or there is no delete family at all if (reader.getTrailer().getEntryCount() == 0 || deleteFamilyCnt == 0) { return false; } if (bloomFilter == null) { return true; } try { if (!bloomFilter.supportsAutoLoading()) { return true; } return bloomFilter.contains(row, rowOffset, rowLen, null); } catch (IllegalArgumentException e) { LOG.error("Bad Delete Family bloom filter data -- proceeding without", e); setDeleteFamilyBloomFilterFaulty(); } return true; } /** * A method for checking Bloom filters. Called directly from * StoreFileScanner in case of a multi-column query. * * @param row * @param rowOffset * @param rowLen * @param col * @param colOffset * @param colLen * @return True if passes */ public boolean passesGeneralBloomFilter(byte[] row, int rowOffset, int rowLen, byte[] col, int colOffset, int colLen) { // Cache Bloom filter as a local variable in case it is set to null by // another thread on an IO error. BloomFilter bloomFilter = this.generalBloomFilter; if (bloomFilter == null) { return true; } byte[] key; switch (bloomFilterType) { case ROW: if (col != null) { throw new RuntimeException("Row-only Bloom filter called with " + "column specified"); } if (rowOffset != 0 || rowLen != row.length) { throw new AssertionError("For row-only Bloom filters the row " + "must occupy the whole array"); } key = row; break; case ROWCOL: key = bloomFilter.createBloomKey(row, rowOffset, rowLen, col, colOffset, colLen); break; default: return true; } // Empty file if (reader.getTrailer().getEntryCount() == 0) return false; try { boolean shouldCheckBloom; ByteBuffer bloom; if (bloomFilter.supportsAutoLoading()) { bloom = null; shouldCheckBloom = true; } else { bloom = reader.getMetaBlock(HFile.BLOOM_FILTER_DATA_KEY, true); shouldCheckBloom = bloom != null; } if (shouldCheckBloom) { boolean exists; // Whether the primary Bloom key is greater than the last Bloom key // from the file info. For row-column Bloom filters this is not yet // a sufficient condition to return false. boolean keyIsAfterLast = lastBloomKey != null && bloomFilter.getComparator().compareFlatKey(key, lastBloomKey) > 0; if (bloomFilterType == BloomType.ROWCOL) { // Since a Row Delete is essentially a DeleteFamily applied to all // columns, a file might be skipped if using row+col Bloom filter. // In order to ensure this file is included an additional check is // required looking only for a row bloom. byte[] rowBloomKey = bloomFilter.createBloomKey(row, 0, row.length, null, 0, 0); if (keyIsAfterLast && bloomFilter.getComparator().compareFlatKey(rowBloomKey, lastBloomKey) > 0) { exists = false; } else { exists = bloomFilter.contains(key, 0, key.length, bloom) || bloomFilter.contains(rowBloomKey, 0, rowBloomKey.length, bloom); } } else { exists = !keyIsAfterLast && bloomFilter.contains(key, 0, key.length, bloom); } return exists; } } catch (IOException e) { LOG.error("Error reading bloom filter data -- proceeding without", e); setGeneralBloomFilterFaulty(); } catch (IllegalArgumentException e) { LOG.error("Bad bloom filter data -- proceeding without", e); setGeneralBloomFilterFaulty(); } return true; } /** * Checks whether the given scan rowkey range overlaps with the current storefile's * @param scan the scan specification. Used to determine the rowkey range. * @return true if there is overlap, false otherwise */ public boolean passesKeyRangeFilter(Scan scan) { if (this.getFirstKey() == null || this.getLastKey() == null) { // the file is empty return false; } if (Bytes.equals(scan.getStartRow(), HConstants.EMPTY_START_ROW) && Bytes.equals(scan.getStopRow(), HConstants.EMPTY_END_ROW)) { return true; } KeyValue smallestScanKeyValue = scan.isReversed() ? KeyValueUtil.createFirstOnRow(scan.getStopRow()) : KeyValueUtil.createFirstOnRow(scan.getStartRow()); KeyValue largestScanKeyValue = scan.isReversed() ? KeyValueUtil.createLastOnRow(scan.getStartRow()) : KeyValueUtil.createLastOnRow(scan.getStopRow()); boolean nonOverLapping = (getComparator().compareFlatKey(this.getFirstKey(), largestScanKeyValue.getKey()) > 0 && !Bytes.equals(scan.isReversed() ? scan.getStartRow() : scan.getStopRow(), HConstants.EMPTY_END_ROW)) || getComparator().compareFlatKey(this.getLastKey(), smallestScanKeyValue.getKey()) < 0; return !nonOverLapping; } public Map<byte[], byte[]> loadFileInfo() throws IOException { Map<byte[], byte[]> fi = reader.loadFileInfo(); byte[] b = fi.get(BLOOM_FILTER_TYPE_KEY); if (b != null) { bloomFilterType = BloomType.valueOf(Bytes.toString(b)); } lastBloomKey = fi.get(LAST_BLOOM_KEY); byte[] cnt = fi.get(DELETE_FAMILY_COUNT); if (cnt != null) { deleteFamilyCnt = Bytes.toLong(cnt); } return fi; } public void loadBloomfilter() { this.loadBloomfilter(BlockType.GENERAL_BLOOM_META); this.loadBloomfilter(BlockType.DELETE_FAMILY_BLOOM_META); } private void loadBloomfilter(BlockType blockType) { try { if (blockType == BlockType.GENERAL_BLOOM_META) { if (this.generalBloomFilter != null) return; // Bloom has been loaded DataInput bloomMeta = reader.getGeneralBloomFilterMetadata(); if (bloomMeta != null) { // sanity check for NONE Bloom filter if (bloomFilterType == BloomType.NONE) { throw new IOException("valid bloom filter type not found in FileInfo"); } else { generalBloomFilter = BloomFilterFactory.createFromMeta(bloomMeta, reader); if (LOG.isTraceEnabled()) { LOG.trace("Loaded " + bloomFilterType.toString() + " " + generalBloomFilter.getClass().getSimpleName() + " metadata for " + reader.getName()); } } } } else if (blockType == BlockType.DELETE_FAMILY_BLOOM_META) { if (this.deleteFamilyBloomFilter != null) return; // Bloom has been loaded DataInput bloomMeta = reader.getDeleteBloomFilterMetadata(); if (bloomMeta != null) { deleteFamilyBloomFilter = BloomFilterFactory.createFromMeta(bloomMeta, reader); LOG.info("Loaded Delete Family Bloom (" + deleteFamilyBloomFilter.getClass().getSimpleName() + ") metadata for " + reader.getName()); } } else { throw new RuntimeException( "Block Type: " + blockType.toString() + "is not supported for Bloom filter"); } } catch (IOException e) { LOG.error("Error reading bloom filter meta for " + blockType + " -- proceeding without", e); setBloomFilterFaulty(blockType); } catch (IllegalArgumentException e) { LOG.error("Bad bloom filter meta " + blockType + " -- proceeding without", e); setBloomFilterFaulty(blockType); } } private void setBloomFilterFaulty(BlockType blockType) { if (blockType == BlockType.GENERAL_BLOOM_META) { setGeneralBloomFilterFaulty(); } else if (blockType == BlockType.DELETE_FAMILY_BLOOM_META) { setDeleteFamilyBloomFilterFaulty(); } } /** * The number of Bloom filter entries in this store file, or an estimate * thereof, if the Bloom filter is not loaded. This always returns an upper * bound of the number of Bloom filter entries. * * @return an estimate of the number of Bloom filter entries in this file */ public long getFilterEntries() { return generalBloomFilter != null ? generalBloomFilter.getKeyCount() : reader.getEntries(); } public void setGeneralBloomFilterFaulty() { generalBloomFilter = null; } public void setDeleteFamilyBloomFilterFaulty() { this.deleteFamilyBloomFilter = null; } public byte[] getLastKey() { return reader.getLastKey(); } public byte[] getLastRowKey() { return reader.getLastRowKey(); } public byte[] midkey() throws IOException { return reader.midkey(); } public long length() { return reader.length(); } public long getTotalUncompressedBytes() { return reader.getTrailer().getTotalUncompressedBytes(); } public long getEntries() { return reader.getEntries(); } public long getDeleteFamilyCnt() { return deleteFamilyCnt; } public byte[] getFirstKey() { return reader.getFirstKey(); } public long indexSize() { return reader.indexSize(); } public BloomType getBloomFilterType() { return this.bloomFilterType; } public long getSequenceID() { return sequenceID; } public void setSequenceID(long sequenceID) { this.sequenceID = sequenceID; } BloomFilter getGeneralBloomFilter() { return generalBloomFilter; } long getUncompressedDataIndexSize() { return reader.getTrailer().getUncompressedDataIndexSize(); } public long getTotalBloomSize() { if (generalBloomFilter == null) return 0; return generalBloomFilter.getByteSize(); } public int getHFileVersion() { return reader.getTrailer().getMajorVersion(); } public int getHFileMinorVersion() { return reader.getTrailer().getMinorVersion(); } public HFile.Reader getHFileReader() { return reader; } void disableBloomFilterForTesting() { generalBloomFilter = null; this.deleteFamilyBloomFilter = null; } public long getMaxTimestamp() { return timeRangeTracker == null ? Long.MAX_VALUE : timeRangeTracker.getMaximumTimestamp(); } } /** * Useful comparators for comparing StoreFiles. */ public abstract static class Comparators { /** * Comparator that compares based on the Sequence Ids of the * the StoreFiles. Bulk loads that did not request a seq ID * are given a seq id of -1; thus, they are placed before all non- * bulk loads, and bulk loads with sequence Id. Among these files, * the size is used to determine the ordering, then bulkLoadTime. * If there are ties, the path name is used as a tie-breaker. */ public static final Comparator<StoreFile> SEQ_ID = Ordering .compound(ImmutableList.of(Ordering.natural().onResultOf(new GetSeqId()), Ordering.natural().onResultOf(new GetFileSize()).reverse(), Ordering.natural().onResultOf(new GetBulkTime()), Ordering.natural().onResultOf(new GetPathName()))); private static class GetSeqId implements Function<StoreFile, Long> { @Override public Long apply(StoreFile sf) { return sf.getMaxSequenceId(); } } private static class GetFileSize implements Function<StoreFile, Long> { @Override public Long apply(StoreFile sf) { return sf.getReader().length(); } } private static class GetBulkTime implements Function<StoreFile, Long> { @Override public Long apply(StoreFile sf) { if (!sf.isBulkLoadResult()) return Long.MAX_VALUE; return sf.getBulkLoadTimestamp(); } } private static class GetPathName implements Function<StoreFile, String> { @Override public String apply(StoreFile sf) { return sf.getPath().getName(); } } } }