org.cloudata.core.tabletserver.TabletMapFile.java Source code

Java tutorial

Introduction

Here is the source code for org.cloudata.core.tabletserver.TabletMapFile.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.cloudata.core.tabletserver;

import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.EOFException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.SortedSet;
import java.util.TreeSet;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.cloudata.core.client.Cell;
import org.cloudata.core.client.CellFilter;
import org.cloudata.core.client.Row;
import org.cloudata.core.common.Constants;
import org.cloudata.core.common.conf.CloudataConf;
import org.cloudata.core.fs.CloudataFileSystem;
import org.cloudata.core.fs.GPath;
import org.cloudata.core.tablet.ColumnValue;
import org.cloudata.core.tablet.TabletInfo;

public class TabletMapFile {
    public static final Log LOG = LogFactory.getLog(TabletMapFile.class.getName());

    public static final String DATA_FILE = "map.dat";
    public static final String IDX_FILE = "map.idx";

    private String fileId;

    protected GPath filePath;

    private SortedSet<MapFileIndexRecord> mapFileIndexRecords = new TreeSet<MapFileIndexRecord>();

    private CloudataConf conf;

    protected CloudataFileSystem fs;

    private TabletInfo tabletInfo;

    private String columnName;

    private int numOfVersion;

    //  private boolean closed = false;
    //private LocatedBlocks dfsBlockInfos;

    public TabletMapFile(CloudataConf conf, CloudataFileSystem fs, TabletInfo tabletInfo, String columnName,
            String fileId, GPath filePath, int numOfVersion) throws IOException {
        this.conf = conf;
        this.fs = fs;
        this.filePath = filePath;
        this.fileId = fileId;
        this.tabletInfo = tabletInfo;
        this.columnName = columnName;
        this.numOfVersion = numOfVersion;
    }

    public TabletMapFile(CloudataConf conf, TabletInfo tabletInfo, String columnName, String fileId,
            int numOfVersion) throws IOException {
        this(conf, CloudataFileSystem.get(conf), tabletInfo, columnName, fileId,
                new GPath(Tablet.getTabletPath(conf, tabletInfo), columnName + "/" + fileId + "/"), numOfVersion);
    }

    public TabletInfo getTabletInfo() {
        return tabletInfo;
    }

    public String getColumnName() {
        return columnName;
    }

    public MapFileWriter getMapFileWriter() throws IOException {
        return getMapFileWriter(0);
    }

    public MapFileWriter getMapFileWriter(int bufferSize) throws IOException {
        int indexBlockSize = conf.getInt("mapfile.indexBlockSize", 128) * 1024;
        return (bufferSize > 0) ? new MapFileWriter(fs, filePath, mapFileIndexRecords, indexBlockSize, bufferSize)
                : new MapFileWriter(fs, filePath, mapFileIndexRecords, indexBlockSize);
    }

    public MapFileReader getMapFileReader(Row.Key startRowKey, Row.Key endRowKey, CellFilter cellFilter)
            throws IOException {
        return new MapFileReader(startRowKey, endRowKey, cellFilter);
    }

    public MapFileReader getMapFileReader(Row.Key startRowKey, Row.Key endRowKey) throws IOException {
        return getMapFileReader(startRowKey, endRowKey, new CellFilter());
    }

    public MapFileReader getMapFileReader(Row.Key startRowKey, Row.Key endRowKey, Cell.Key cellKey)
            throws IOException {
        CellFilter cellFilter = new CellFilter();
        cellFilter.setCellKey(cellKey);
        return getMapFileReader(startRowKey, endRowKey, cellFilter);
    }

    public MapFileReader getMapFileReader() throws IOException {
        return getMapFileReader(new CellFilter());
    }

    public MapFileReader getMapFileReader(CellFilter cellFilter) throws IOException {
        return getMapFileReader(Row.Key.MIN_KEY, Row.Key.MAX_KEY, cellFilter);
    }

    public boolean loadIndex() throws IOException {
        DataInputStream in = null;

        GPath idxPath = new GPath(filePath, IDX_FILE);
        GPath dataPath = new GPath(filePath, DATA_FILE);
        try {
            int fileCount = 0;
            if (fs.exists(idxPath))
                fileCount++;
            if (fs.exists(dataPath))
                fileCount++;
            if (fileCount == 0) {
                return false;
            } else if (fileCount == 1) {
                LOG.error("data or idx file not exists. tabletInfo=" + tabletInfo + ", column=" + columnName
                        + ", fileId=" + fileId);
                return false;
            }

            long dataFileLength = fs.getLength(new GPath(filePath, DATA_FILE));
            boolean idxFileError = false;
            long lastIndexOffset = 0;

            //LOG.debug("loadIndex:" + idxPath);
            in = new DataInputStream(fs.open(idxPath));
            MapFileIndexRecord mapFileIndexRecord = null;
            try {
                while (true) {
                    mapFileIndexRecord = new MapFileIndexRecord();
                    mapFileIndexRecord.read(in);
                    if (mapFileIndexRecord.getOffset() > dataFileLength) {
                        idxFileError = true;
                        lastIndexOffset = mapFileIndexRecord.getOffset();
                        break;
                    }
                    mapFileIndexRecords.add(mapFileIndexRecord);
                }
            } catch (EOFException eof) {

            }

            if (idxFileError) {
                throw new IOException("Can't load map file. index offset(" + lastIndexOffset
                        + ") greate than data file length(" + dataFileLength + ").[path=" + filePath + "]");
            }
        } catch (Exception e) {
            e.printStackTrace();
            throw new IOException(e.getMessage());
        } finally {
            if (in != null)
                in.close();
        }

        //    try {
        //      synchronized(this) {
        //        dfsBlockInfos = fs.getBlockInfos(dataPath);
        //      }
        //    } catch (Exception e) {
        //      e.printStackTrace();
        //    }
        return true;
    }

    public long getIndexMemorySize() throws IOException {
        long result = 0;
        for (MapFileIndexRecord mapFileIndexRecord : mapFileIndexRecords) {
            result += mapFileIndexRecord.getMemorySize();
        }
        return result;
    }

    class MapFileReader implements Scanner, Searchable {
        private Row.Key currentRowKey;
        private DataInputStream dataIn;
        private DataInputStream seekableIn;

        private Row.Key startRowKey;
        private Row.Key endRowKey;

        private CellFilter cellFilter;

        private GPath mapFilePath;

        public MapFileReader(Row.Key startRowKey, Row.Key endRowKey, CellFilter cellFilter) throws IOException {
            this.startRowKey = startRowKey;
            this.endRowKey = endRowKey;
            this.cellFilter = cellFilter;

            MapFileIndexRecord indexRecord = findNearest(startRowKey, cellFilter.getTargetCellKey());

            if (indexRecord != null) {
                mapFilePath = new GPath(filePath, DATA_FILE);
                try {
                    seekableIn = fs.openDataInputStream(mapFilePath);
                } catch (IOException e) {
                    LOG.warn("Error in creating MapFileReader: " + filePath + ": " + e.getMessage());
                    throw e;
                }
                fs.seek(seekableIn, indexRecord.getOffset());
                dataIn = seekableIn;
            } else {
                seekableIn = null;
            }
        }

        public ColumnValue next() throws IOException {
            if (dataIn == null) {
                return null;
            }
            while (true) {
                MapFileColumnValue mapFileColumnValue = new MapFileColumnValue();
                try {
                    mapFileColumnValue.readFields(dataIn);
                } catch (EOFException eof) {
                    return null;
                }
                currentRowKey = mapFileColumnValue.getRowKey();

                if (currentRowKey.compareTo(endRowKey) > 0) {
                    return null;
                }

                if (currentRowKey.compareTo(startRowKey) < 0) {
                    mapFileColumnValue = skipToTargetRow();
                    if (mapFileColumnValue == null) {
                        currentRowKey = null;
                        return null;
                    }
                    currentRowKey = mapFileColumnValue.getRowKey();
                }

                if (cellFilter.matchKey(mapFileColumnValue.getCellKey())) {
                    ColumnValue columnValue = new ColumnValue();
                    copyColumnValue(mapFileColumnValue, columnValue);
                    return columnValue;
                }
            }
        }

        private MapFileColumnValue skipToTargetRow() throws IOException {
            MapFileColumnValue mapFileColumnValue = new MapFileColumnValue();
            try {
                while (true) {
                    mapFileColumnValue.readFields(dataIn);
                    Row.Key rowKey = mapFileColumnValue.getRowKey();

                    if (rowKey.compareTo(startRowKey) < 0) {
                        continue;
                    } else if (rowKey.compareTo(startRowKey) >= 0 && rowKey.compareTo(endRowKey) <= 0) {
                        return mapFileColumnValue;
                    } else if (rowKey.compareTo(endRowKey) > 0) {
                        return null;
                    }
                }
            } catch (EOFException eof) {
            }
            return null;
        }

        public Row.Key getCurrentRowKey() {
            return currentRowKey;
        }

        public void close() throws IOException {
            if (dataIn != null)
                dataIn.close();
            dataIn = null;
        }

        public int avaliable() throws IOException {
            if (dataIn == null)
                return 0;
            return dataIn.available();
        }
    }

    public static class MapFileWriter {
        private DataOutputStream dataOut;
        private DataOutputStream indexOut;
        private CloudataFileSystem fs;

        private ColumnValue lastColumnRecord;

        private int blockWriteBytes = 0;
        private long currentDataFilePos = 0;
        private long indexFileOffset = 0;

        //FIXME    ? 
        private int indexBlockSize;
        private final int BLOCK_RECORD_COUNT = 300;

        private int blockWriteRecordCount = 0;

        private long sumDataLength = 0;
        private long sumIndexLength = 0;

        //private long writeTotalTime = 0;
        SortedSet<MapFileIndexRecord> mapFileIndexRecords;

        public MapFileWriter(CloudataFileSystem fs, GPath filePath,
                SortedSet<MapFileIndexRecord> mapFileIndexRecords, int indexBlockSize) throws IOException {
            this(fs, filePath, mapFileIndexRecords, indexBlockSize, 0);
        }

        public MapFileWriter(CloudataFileSystem fs, GPath filePath,
                SortedSet<MapFileIndexRecord> mapFileIndexRecords, int indexBlockSize, int outputBufferSize)
                throws IOException {
            this.indexBlockSize = indexBlockSize;
            this.fs = fs;
            this.mapFileIndexRecords = mapFileIndexRecords;
            fs.mkdirs(filePath);

            dataOut = (outputBufferSize > 0)
                    ? new DataOutputStream(fs.create(new GPath(filePath, DATA_FILE), outputBufferSize))
                    : new DataOutputStream(fs.create(new GPath(filePath, DATA_FILE)));
            indexOut = new DataOutputStream(fs.create(new GPath(filePath, IDX_FILE)));
        }

        public int write(ValueCollection columnValues) throws IOException {
            if (columnValues.isEmpty()) {
                return 0;
            }
            ColumnValue columnValue = columnValues.getWithDeleted();

            //long startTime = System.currentTimeMillis();
            int writeSize = columnValues.write(dataOut);
            //writeTotalTime += (System.currentTimeMillis() - startTime);
            sumDataLength += writeSize;

            makeFileIndex(writeSize, columnValue, indexBlockSize);

            lastColumnRecord = columnValue;

            return writeSize;
        }

        public int write(ColumnValue columnValue) throws IOException {
            MapFileColumnValue mapFileColumnValue = new MapFileColumnValue();
            ValueCollection.copyMapFileColumnValue(columnValue, mapFileColumnValue);
            mapFileColumnValue.write(dataOut);
            int writeSize = mapFileColumnValue.size();

            makeFileIndex(writeSize, columnValue, indexBlockSize);

            lastColumnRecord = columnValue;
            //dataOut.flush();
            //indexOut.flush();
            return writeSize;
        }

        private void makeFileIndex(int writeSize, ColumnValue columnValue, int checkSize) throws IOException {
            blockWriteBytes += writeSize;
            currentDataFilePos += writeSize;
            blockWriteRecordCount++;

            //?? writeBytes block size(64kb) ?  ?  index? 
            if (blockWriteBytes > checkSize || blockWriteRecordCount > BLOCK_RECORD_COUNT) {
                //index ?? index  
                MapFileIndexRecord mapFileIndexRecord = new MapFileIndexRecord(columnValue.getRowKey(), columnValue,
                        indexFileOffset);
                mapFileIndexRecord.write(indexOut);
                mapFileIndexRecords.add(mapFileIndexRecord);

                sumIndexLength += mapFileIndexRecord.getMemorySize();

                blockWriteBytes = 0;
                blockWriteRecordCount = 0;
                indexFileOffset = currentDataFilePos;
            }
        }

        public void close() throws IOException {
            if (dataOut == null) {
                return;
            }
            //      if (closed) {
            //        return;
            //      }

            if (blockWriteBytes > 0) {
                makeFileIndex(0, lastColumnRecord, 0);
            }

            dataOut.flush();
            indexOut.flush();

            IOException exception = null;

            if (dataOut != null) {
                try {
                    dataOut.close();
                    dataOut = null;
                    //fs.setReplication(new GPath(filePath, DATA_FILE), (short)3);
                } catch (IOException e) {
                    LOG.error(e.getMessage() + ", sumDataLength=" + sumDataLength, e);
                    exception = e;
                }
            }
            if (indexOut != null) {
                try {
                    indexOut.close();
                    indexOut = null;
                } catch (IOException e) {
                    LOG.error(e.getMessage() + ", sumIndexLength=" + sumIndexLength, e);
                    exception = e;
                }
            }
            //      closed = true;

            if (exception != null) {
                throw exception;
            }
        }
    }

    public ValueCollection scan(Row.Key rowKey, Cell.Key columnKey) throws IOException {
        MapFileReader reader = null;
        try {
            reader = getMapFileReader(rowKey, new Row.Key(Constants.MAX_VALUE), columnKey);

            ValueCollection valueCollection = new ValueCollection();
            ColumnValue columnValue = null;

            while ((columnValue = reader.next()) != null) {
                Row.Key scanRowKey = columnValue.getRowKey();
                if (scanRowKey.compareTo(rowKey) > 0) {
                    break;
                }

                if (scanRowKey.equals(rowKey)) {
                    if (columnKey == null) {
                        valueCollection.add(columnValue, numOfVersion);
                    } else {
                        if (columnValue.getCellKey().compareTo(columnKey) == 0) {
                            valueCollection.add(columnValue, numOfVersion);
                        } else if (columnValue.getCellKey().compareTo(columnKey) > 0) {
                            break;
                        } else {

                        }
                    }
                }
            }
            return valueCollection;
        } finally {
            reader.close();
        }
    }

    public List<ColumnValue> scan(Row.Key rowKey) throws IOException {
        MapFileReader reader = null;
        try {
            reader = getMapFileReader(rowKey, rowKey);

            List<ColumnValue> result = new ArrayList<ColumnValue>();
            ColumnValue columnValue = null;

            while ((columnValue = reader.next()) != null) {
                result.add(columnValue);
            }
            return result;
        } finally {
            if (reader != null) {
                reader.close();
            }
        }
    }

    public boolean hasValue(Row.Key rowKey, Cell.Key cellKey) throws IOException {
        MapFileReader reader = null;
        try {
            reader = getMapFileReader(rowKey, rowKey, cellKey);

            ColumnValue columnValue = null;

            while ((columnValue = reader.next()) != null) {
                if (columnValue.getRowKey().compareTo(rowKey) < 0) {
                    continue;
                }

                if (cellKey != null && columnValue.getCellKey().compareTo(cellKey) < 0) {
                    continue;
                }

                if (columnValue.getRowKey().compareTo(rowKey) > 0) {
                    return false;
                } else {
                    if (cellKey != null && columnValue.getCellKey().compareTo(cellKey) > 0) {
                        return false;
                    }
                    return true;
                }
            }
            return false;
        } finally {
            reader.close();
        }
    }

    public MapFileIndexRecord findNearest(Row.Key targetRowKey, Cell.Key cellKey) throws IOException {
        SortedSet<MapFileIndexRecord> tailMap = mapFileIndexRecords
                .tailSet(new MapFileIndexRecord(targetRowKey, cellKey));
        if (tailMap.size() > 0) {
            return tailMap.first();
        } else {
            return null;
        }
    }

    public GPath getFilePath() {
        return filePath;
    }

    public void moveTo(GPath targetPath, boolean deleteParent) throws IOException {
        GPath originPath = this.filePath;
        GPath originMapDataPath = new GPath(originPath, DATA_FILE);
        GPath originMapIndexPath = new GPath(originPath, IDX_FILE);

        GPath destDataPath = new GPath(targetPath, DATA_FILE);
        GPath destIndexPath = new GPath(targetPath, IDX_FILE);

        //LOG.info("move data file: " + originMapDataPath + " to " + destDataPath);
        if (!fs.move(originMapDataPath, destDataPath)) {
            LOG.error("Can't move data file: " + originMapDataPath + " to " + destDataPath);
            throw new IOException("Can't move data file: " + originMapDataPath + " to " + destDataPath);
        }

        //LOG.debug("move index file: " + originMapIndexPath + " to " + destIndexPath);
        if (!fs.move(originMapIndexPath, destIndexPath)) {
            LOG.error("Can't move index file: " + originMapIndexPath + " to " + destIndexPath);
            throw new IOException("Can't move index file: " + originMapIndexPath + " to " + destIndexPath);
        }

        //delete temp dir
        if (deleteParent) {
            int retry = 0;
            while (retry < 5) {
                //LOG.debug("minor compaction temp dir:" + originPath);
                boolean result = fs.delete(originPath, true);
                if (result) {
                    break;
                }

                LOG.info("fail to delete file [" + originPath + "] but retry");

                retry++;
                try {
                    Thread.sleep(1000);
                } catch (InterruptedException e) {
                    return;
                }
            }
            if (retry >= 5) {
                LOG.warn("Can't delete minor compaction temp dir:" + originPath);
            }
            //LOG.debug("moveTo:delete:" + originMapIndexPath + " to " + destIndexPath);
        }
        this.filePath = targetPath;
    }

    public boolean delete() throws IOException {
        int retry = 0;
        while (retry < 5) {
            boolean result = fs.delete(filePath, true);
            if (result) {
                return true;
            }
            try {
                Thread.sleep(500);
            } catch (InterruptedException e) {
            }
        }
        return false;
    }

    public boolean equals(Object obj) {
        if (!(obj instanceof TabletMapFile))
            return false;

        TabletMapFile otherObj = (TabletMapFile) obj;

        if (!tabletInfo.getTabletName().equals(otherObj.tabletInfo.getTabletName()))
            return false;
        if (!columnName.equals(otherObj.columnName))
            return false;
        return fileId.equals(otherObj.fileId);
    }

    public String toString() {
        return "[tabletName=" + tabletInfo.getTabletName() + ", columnName=" + columnName + ", path=" + filePath;
    }

    //  public void printIndex() throws IOException {
    //    System.out.println("\t\tMapFile: " + filePath);
    //    for(MapFileIndexRecord indexRecord: mapFileIndexRecords) {
    //      System.out.println("\t\t\t" + indexRecord);
    //    }
    //  }

    public SortedSet<MapFileIndexRecord> getMapFileIndexRecords() {
        return mapFileIndexRecords;
    }

    public long getDataFileSize() throws IOException {
        return fs.getLength(new GPath(filePath, DATA_FILE));
    }

    /**
     * @return the fileId
     */
    public String getFileId() {
        return fileId;
    }

    public CloudataFileSystem getFileSystem() {
        return fs;
    }

    public static void copyColumnValue(MapFileColumnValue mapFileColumnValue, ColumnValue columnValue) {
        columnValue.setRowKey(mapFileColumnValue.getRowKey());
        columnValue.setCellKey(mapFileColumnValue.getCellKey());
        columnValue.setValue(mapFileColumnValue.getValue());
        columnValue.setTimestamp(mapFileColumnValue.getTimestamp());
        columnValue.setDeleted(mapFileColumnValue.isDeleted());
    }

    public static void main(String[] args) throws IOException {
        if (args.length < 4) {
            System.out.println("Usage: java TabletMapFile <table name> <tablet name> <column name> <file id>");
            System.exit(0);
        }

        TabletInfo tabletInfo = new TabletInfo();
        tabletInfo.setTableName(args[0]);
        tabletInfo.setTabletName(args[1]);

        TabletMapFile mapFile = new TabletMapFile(new CloudataConf(), tabletInfo, args[2], args[3], 0);

        mapFile.loadIndex();

        MapFileReader reader = mapFile.getMapFileReader(Row.Key.MIN_KEY, Row.Key.MAX_KEY);
        try {
            ColumnValue columnValue = null;
            while ((columnValue = reader.next()) != null) {
                System.out.println(columnValue.getRowKey() + "," + columnValue.getCellKey() + ","
                        + new String(columnValue.getValue()));
                /*
                try {
                TabletInfo tabletInfo1 = new TabletInfo();
                if(columnValue.getValue() == null) {
                   System.out.println("[" + columnValue.getRowKey() + "][" + columnValue.getCellKey() + 
                          "][" + columnValue.getTimestamp() + "][" + columnValue.isDeleted() + "][" + 
                          (columnValue.getValue() == null ? "null" : columnValue.getValue().length) + "]");
                continue;
                }
                tabletInfo1.readFields(columnValue.getValue());
                  System.out.println("[" + columnValue.getRowKey() + "][" + columnValue.getCellKey() + 
                      "][" + columnValue.getTimestamp() + "][" + columnValue.isDeleted() + "][" + 
                    (columnValue.getValue() == null ? "null" : columnValue.getValue().length) + "]" + tabletInfo1);
                } catch (Exception e) {
                System.out.println("Error:" + columnValue.getRowKey() + "][" + columnValue.getCellKey());
                }
                */
            }
        } finally {
            reader.close();
        }
    }
}