com.bah.lucene.BlockCacheDirectoryFactoryV2.java Source code

Java tutorial

Introduction

Here is the source code for com.bah.lucene.BlockCacheDirectoryFactoryV2.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.bah.lucene;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.lucene.store.Directory;

import com.bah.lucene.blockcache_v2.*;
import com.bah.lucene.blockcache_v2.BaseCache.STORE;

import java.io.IOException;
import java.text.MessageFormat;
import java.util.*;
import java.util.Map.Entry;

import static com.bah.lucene.util.BlurConstants.*;

public class BlockCacheDirectoryFactoryV2 extends BlockCacheDirectoryFactory {

    private static final Log LOG = LogFactory.getLog(BlockCacheDirectoryFactoryV2.class);

    private final Cache _cache;

    public BlockCacheDirectoryFactoryV2(Configuration configuration, long totalNumberOfBytes) {

        final int fileBufferSizeInt = configuration.getInt(BLUR_SHARD_BLOCK_CACHE_V2_FILE_BUFFER_SIZE, 8192);
        LOG.info(MessageFormat.format("{0}={1}", BLUR_SHARD_BLOCK_CACHE_V2_FILE_BUFFER_SIZE, fileBufferSizeInt));
        final int cacheBlockSizeInt = configuration.getInt(BLUR_SHARD_BLOCK_CACHE_V2_CACHE_BLOCK_SIZE, 8192);
        LOG.info(MessageFormat.format("{0}={1}", BLUR_SHARD_BLOCK_CACHE_V2_CACHE_BLOCK_SIZE, cacheBlockSizeInt));

        final Map<String, Integer> cacheBlockSizeMap = new HashMap<String, Integer>();
        for (Entry<String, String> prop : configuration) {
            String key = prop.getKey();
            if (key.startsWith(BLUR_SHARD_BLOCK_CACHE_V2_CACHE_BLOCK_SIZE_PREFIX)) {
                String value = prop.getValue();
                int cacheBlockSizeForFile = Integer.parseInt(value);
                String fieldType = key.substring(BLUR_SHARD_BLOCK_CACHE_V2_CACHE_BLOCK_SIZE_PREFIX.length());

                cacheBlockSizeMap.put(fieldType, cacheBlockSizeForFile);
                LOG.info(
                        MessageFormat.format("{0}={1} for file type [{2}]", key, cacheBlockSizeForFile, fieldType));
            }
        }

        final STORE store = STORE.valueOf(configuration.get(BLUR_SHARD_BLOCK_CACHE_V2_STORE, OFF_HEAP));
        LOG.info(MessageFormat.format("{0}={1}", BLUR_SHARD_BLOCK_CACHE_V2_STORE, store));

        final Set<String> cachingFileExtensionsForRead = getSet(
                configuration.get(BLUR_SHARD_BLOCK_CACHE_V2_READ_CACHE_EXT, DEFAULT_VALUE));
        LOG.info(MessageFormat.format("{0}={1}", BLUR_SHARD_BLOCK_CACHE_V2_READ_CACHE_EXT,
                cachingFileExtensionsForRead));

        final Set<String> nonCachingFileExtensionsForRead = getSet(
                configuration.get(BLUR_SHARD_BLOCK_CACHE_V2_READ_NOCACHE_EXT, DEFAULT_VALUE));
        LOG.info(MessageFormat.format("{0}={1}", BLUR_SHARD_BLOCK_CACHE_V2_READ_NOCACHE_EXT,
                nonCachingFileExtensionsForRead));

        final boolean defaultReadCaching = configuration.getBoolean(BLUR_SHARD_BLOCK_CACHE_V2_READ_DEFAULT, true);
        LOG.info(MessageFormat.format("{0}={1}", BLUR_SHARD_BLOCK_CACHE_V2_READ_DEFAULT, defaultReadCaching));

        final Set<String> cachingFileExtensionsForWrite = getSet(
                configuration.get(BLUR_SHARD_BLOCK_CACHE_V2_WRITE_CACHE_EXT, DEFAULT_VALUE));
        LOG.info(MessageFormat.format("{0}={1}", BLUR_SHARD_BLOCK_CACHE_V2_WRITE_CACHE_EXT,
                cachingFileExtensionsForWrite));

        final Set<String> nonCachingFileExtensionsForWrite = getSet(
                configuration.get(BLUR_SHARD_BLOCK_CACHE_V2_WRITE_NOCACHE_EXT, DEFAULT_VALUE));
        LOG.info(MessageFormat.format("{0}={1}", BLUR_SHARD_BLOCK_CACHE_V2_WRITE_NOCACHE_EXT,
                nonCachingFileExtensionsForWrite));

        final boolean defaultWriteCaching = configuration.getBoolean(BLUR_SHARD_BLOCK_CACHE_V2_WRITE_DEFAULT, true);
        LOG.info(MessageFormat.format("{0}={1}", BLUR_SHARD_BLOCK_CACHE_V2_WRITE_DEFAULT, defaultWriteCaching));

        Size fileBufferSize = new Size() {
            @Override
            public int getSize(CacheDirectory directory, String fileName) {
                return fileBufferSizeInt;
            }
        };

        Size cacheBlockSize = new Size() {
            @Override
            public int getSize(CacheDirectory directory, String fileName) {
                String ext = getExt(fileName);
                Integer size = cacheBlockSizeMap.get(ext);
                if (size != null) {
                    return size;
                }
                return cacheBlockSizeInt;
            }
        };

        FileNameFilter readFilter = new FileNameFilter() {
            @Override
            public boolean accept(CacheDirectory directory, String fileName) {
                String ext = getExt(fileName);
                if (cachingFileExtensionsForRead.contains(ext)) {
                    return true;
                } else if (nonCachingFileExtensionsForRead.contains(ext)) {
                    return false;
                }
                return defaultReadCaching;
            }
        };

        FileNameFilter writeFilter = new FileNameFilter() {
            @Override
            public boolean accept(CacheDirectory directory, String fileName) {
                String ext = getExt(fileName);
                if (cachingFileExtensionsForWrite.contains(ext)) {
                    return true;
                } else if (nonCachingFileExtensionsForWrite.contains(ext)) {
                    return false;
                }
                return defaultWriteCaching;
            }
        };

        Quiet quiet = new Quiet() {
            @Override
            public boolean shouldBeQuiet(CacheDirectory directory, String fileName) {
                Thread thread = Thread.currentThread();
                String name = thread.getName();
                if (name.startsWith(SHARED_MERGE_SCHEDULER)) {
                    return true;
                }
                return false;
            }
        };

        _cache = new BaseCache(totalNumberOfBytes, fileBufferSize, cacheBlockSize, readFilter, writeFilter, quiet,
                store);
    }

    private Set<String> getSet(String value) {
        String[] split = value.split(",");
        return new HashSet<String>(Arrays.asList(split));
    }

    @Override
    public Directory newDirectory(String table, String shard, Directory directory,
            Set<String> tableBlockCacheFileTypes) throws IOException {
        return new CacheDirectory(table, shard, directory, _cache, tableBlockCacheFileTypes);
    }

    private static String getExt(String fileName) {
        int indexOf = fileName.lastIndexOf('.');
        if (indexOf < 0) {
            return DEFAULT_VALUE;
        }
        return fileName.substring(indexOf + 1);
    }

}