com.bigdata.dastor.io.SSTable.java Source code

Introduction

Here is the source code for com.bigdata.dastor.io.SSTable.java

Source

package com.bigdata.dastor.io;
/*
 * 
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 * 
 */

import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Arrays;

import org.apache.log4j.Logger;
import org.apache.commons.lang.StringUtils;

import com.bigdata.dastor.db.DecoratedKey;
import com.bigdata.dastor.dht.IPartitioner;
import com.bigdata.dastor.io.util.FileUtils;
import com.bigdata.dastor.utils.BloomFilter;

/**
 * This class is built on top of the SequenceFile. It stores
 * data on disk in sorted fashion. However the sorting is upto
 * the application. This class expects keys to be handed to it
 * in sorted order.
 *
 * A separate index file is maintained as well, containing the
 * SSTable keys and the offset into the SSTable at which they are found.
 * Every 1/indexInterval key is read into memory when the SSTable is opened.
 *
 * Finally, a bloom filter file is also kept for the keys in each SSTable.
 */
public abstract class SSTable {
    static final Logger logger = Logger.getLogger(SSTable.class);

    public static final int FILES_ON_DISK = 3; // data, index, and bloom filter

    protected String path;
    protected IPartitioner partitioner;
    protected BloomFilter bf;
    protected String columnFamilyName;
    protected IndexSummary indexSummary;

    public static final String TEMPFILE_MARKER = "tmp";

    public SSTable(String filename, IPartitioner partitioner) {
        assert filename.endsWith("-Data.db");
        columnFamilyName = parseColumnFamilyName(filename);
        this.path = filename;
        this.partitioner = partitioner;
    }

    protected static String parseColumnFamilyName(String filename) {
        return new File(filename).getName().split("-")[0];
    }

    public static String indexFilename(String dataFile) {
        String[] parts = dataFile.split("-");
        parts[parts.length - 1] = "Index.db";
        return StringUtils.join(parts, "-");
    }

    public String indexFilename() {
        return indexFilename(path);
    }

    protected static String compactedFilename(String dataFile) {
        String[] parts = dataFile.split("-");
        parts[parts.length - 1] = "Compacted";
        return StringUtils.join(parts, "-");
    }

    /**
     * We use a ReferenceQueue to manage deleting files that have been compacted
     * and for which no more SSTable references exist.  But this is not guaranteed
     * to run for each such file because of the semantics of the JVM gc.  So,
     * we write a marker to `compactedFilename` when a file is compacted;
     * if such a marker exists on startup, the file should be removed.
     *
     * @return true if the file was deleted
     */
    public static boolean deleteIfCompacted(String dataFilename) throws IOException {
        if (new File(compactedFilename(dataFilename)).exists()) {
            FileUtils.deleteWithConfirm(new File(dataFilename));
            FileUtils.deleteWithConfirm(new File(SSTable.indexFilename(dataFilename)));
            FileUtils.deleteWithConfirm(new File(SSTable.filterFilename(dataFilename)));
            FileUtils.deleteWithConfirm(new File(SSTable.compactedFilename(dataFilename)));
            logger.info("Deleted " + dataFilename);
            return true;
        }
        return false;
    }

    protected String compactedFilename() {
        return compactedFilename(path);
    }

    protected static String filterFilename(String dataFile) {
        String[] parts = dataFile.split("-");
        parts[parts.length - 1] = "Filter.db";
        return StringUtils.join(parts, "-");
    }

    public String filterFilename() {
        return filterFilename(path);
    }

    public String getFilename() {
        return path;
    }

    /** @return full paths to all the files associated w/ this SSTable */
    public List<String> getAllFilenames() {
        // TODO streaming relies on the -Data (getFilename) file to be last, this is clunky
        return Arrays.asList(indexFilename(), filterFilename(), getFilename());
    }

    public String getColumnFamilyName() {
        return columnFamilyName;
    }

    public String getTableName() {
        return parseTableName(path);
    }

    public static String parseTableName(String filename) {
        return new File(filename).getParentFile().getName();
    }

    public static long getTotalBytes(Iterable<SSTableReader> sstables) {
        long sum = 0;
        for (SSTableReader sstable : sstables) {
            sum += sstable.length();
        }
        return sum;
    }

    public long bytesOnDisk() {
        long bytes = 0;
        for (String fname : getAllFilenames()) {
            bytes += new File(fname).length();
        }
        return bytes;
    }

    @Override
    public String toString() {
        return getClass().getName() + "(" + "path='" + path + '\'' + ')';
    }

    public static class PositionSize {
        public final long position;
        public final long size;

        public PositionSize(long position, long size) {
            this.position = position;
            this.size = size;
        }
    }
}