com.bah.lucene.BlockCacheDirectoryFactoryV1.java Source code

Introduction

Here is the source code for com.bah.lucene.BlockCacheDirectoryFactoryV1.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.bah.lucene;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.lucene.store.Directory;

import com.bah.lucene.blockcache.BlockCache;
import com.bah.lucene.blockcache.BlockDirectory;
import com.bah.lucene.blockcache.BlockDirectoryCache;
import com.bah.lucene.blockcache.Cache;

import java.io.IOException;
import java.text.MessageFormat;
import java.util.Set;

import static com.bah.lucene.util.BlurConstants.BLUR_SHARD_BLOCKCACHE_DIRECT_MEMORY_ALLOCATION;
import static com.bah.lucene.util.BlurConstants.BLUR_SHARD_BLOCKCACHE_SLAB_COUNT;

public class BlockCacheDirectoryFactoryV1 extends BlockCacheDirectoryFactory {

    private static final Log LOG = LogFactory.getLog(BlockCacheDirectoryFactoryV1.class);

    private final Cache _cache;

    public BlockCacheDirectoryFactoryV1(Configuration configuration, long totalNumberOfBytes) {
        // setup block cache
        // 134,217,728 is the slab size, therefore there are 16,384 blocks
        // in a slab when using a block size of 8,192
        int numberOfBlocksPerSlab = 16384;
        int blockSize = BlockDirectory.BLOCK_SIZE;
        int slabCount = configuration.getInt(BLUR_SHARD_BLOCKCACHE_SLAB_COUNT, -1);
        slabCount = getSlabCount(slabCount, numberOfBlocksPerSlab, blockSize, totalNumberOfBytes);
        Cache cache;
        if (slabCount >= 1) {
            BlockCache blockCache;
            boolean directAllocation = configuration.getBoolean(BLUR_SHARD_BLOCKCACHE_DIRECT_MEMORY_ALLOCATION,
                    true);

            int slabSize = numberOfBlocksPerSlab * blockSize;
            LOG.info(MessageFormat.format(
                    "Number of slabs of block cache [{0}] with direct memory allocation set to [{1}]", slabCount,
                    directAllocation));
            LOG.info(MessageFormat.format(
                    "Block cache target memory usage, slab size of [{0}] will allocate [{1}] slabs and use ~[{2}] bytes",
                    slabSize, slabCount, ((long) slabCount * (long) slabSize)));

            try {
                long totalMemory = (long) slabCount * (long) numberOfBlocksPerSlab * (long) blockSize;
                blockCache = new BlockCache(directAllocation, totalMemory, slabSize);
            } catch (OutOfMemoryError e) {
                if ("Direct buffer memory".equals(e.getMessage())) {
                    System.err.println(
                            "The max direct memory is too low.  Either increase by setting (-XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages) or disable direct allocation by (blur.shard.blockcache.direct.memory.allocation=false) in blur-site.properties");
                    System.exit(1);
                }
                throw e;
            }
            cache = new BlockDirectoryCache(blockCache);
        } else {
            cache = BlockDirectory.NO_CACHE;
        }
        _cache = cache;
    }

    @Override
    public Directory newDirectory(String table, String shard, Directory directory, Set<String> blockCacheFileTypes)
            throws IOException {
        return new BlockDirectory(table + "_" + shard, directory, _cache, blockCacheFileTypes);
    }

    private static int getSlabCount(int slabCount, int numberOfBlocksPerSlab, int blockSize,
            long totalNumberOfBytes) {
        if (slabCount < 0) {
            long slabSize = numberOfBlocksPerSlab * blockSize;
            if (totalNumberOfBytes < slabSize) {
                throw new RuntimeException(
                        "Auto slab setup cannot happen, JVM option -XX:MaxDirectMemorySize not set.");
            }
            return (int) (totalNumberOfBytes / slabSize);
        }
        return slabCount;
    }

}