org.carbondata.query.util.CacheUtil.java Source code

Java tutorial

Introduction

Here is the source code for org.carbondata.query.util.CacheUtil.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.carbondata.query.util;

import java.io.DataInputStream;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.carbondata.common.logging.LogService;
import org.carbondata.common.logging.LogServiceFactory;
import org.carbondata.core.carbon.CarbonDef;
import org.carbondata.core.constants.CarbonCommonConstants;
import org.carbondata.core.datastorage.store.filesystem.CarbonFile;
import org.carbondata.core.datastorage.store.impl.FileFactory;
import org.carbondata.core.datastorage.store.impl.FileFactory.FileType;
import org.carbondata.core.keygenerator.KeyGenerator;
import org.carbondata.core.metadata.CarbonMetadata.Dimension;
import org.carbondata.core.metadata.CarbonSchemaReader;
import org.carbondata.core.util.CarbonProperties;
import org.carbondata.core.util.CarbonUtil;
import org.carbondata.query.datastorage.Member;
import org.carbondata.query.queryinterface.filter.CarbonFilterInfo;
import org.carbondata.query.wrappers.ArrayWrapper;

import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
import org.apache.commons.codec.binary.Base64;

//import org.pentaho.platform.util.logging.Logger;

/**
 * Class Description : This class will be used for prepare member and
 * hierarchies cache , it will read member and hierarchies files to prepare
 * cache Dimension: Filename = DimensionName Columns = 2 Rows =
 * Value(String),SurrogateKey(Long) Hierarchy: Filename= HierachyName Content =
 * byte content Fact/Aggregate: Filename= TableName Columns = dimensions +
 * measures Content = ; separated values
 * Version 1.0
 */

public final class CacheUtil {
    /**
     * LOGGER
     */
    private static final LogService LOGGER = LogServiceFactory.getLogService(CacheUtil.class.getName());
    /**
     * Map intiaL capacity
     */
    private static final int SET_INTIAL_CAPECITY = 1000;
    /**
     * list initial capacity
     */
    private static final int LIST_INTIAL_CAPECITY = 10;

    private CacheUtil() {

    }

    /**
     * This method Read dimension files and prepare Int2ObjectMap for dimension
     * which will contain all the member of dimension
     * Dimension: Filename = DimensionName Columns = 2 Rows =
     * Value(String),SurrogateKey(Long)
     *
     * @param filesLocaton    dimension file location
     * @param namecolumnindex name column index
     * @return Int2ObjectMap<Member> it will contain all the members
     */
    public static Member[][] getMembersList(String filesLocaton, byte nameColumnIndex, String dataType) {
        LOGGER.debug("Reading members data from location: " + filesLocaton);
        //        File decryptedFile = decryptEncyptedFile(filesLocaton);
        Member[][] members = processMemberFile(filesLocaton, dataType);
        return members;
    }

    public static int getMinValue(String filesLocaton) {
        if (null == filesLocaton) {
            return 0;
        }
        DataInputStream fileChannel = null;
        try {
            if (!FileFactory.isFileExist(filesLocaton, FileFactory.getFileType(filesLocaton))) {
                return 0;
            }
            fileChannel = new DataInputStream(
                    FileFactory.getDataInputStream(filesLocaton, FileFactory.getFileType(filesLocaton), 10240));
            fileChannel.readInt();
            return fileChannel.readInt();
        } catch (IOException e) {
            //            e.printStackTrace();
            LOGGER.error(e, e.getMessage());
        } finally {
            CarbonUtil.closeStreams(fileChannel);
        }
        return 0;
    }

    public static int getMinValueFromLevelFile(String filesLocaton) {
        if (null == filesLocaton) {
            return 0;
        }
        DataInputStream fileChannel = null;
        try {
            if (!FileFactory.isFileExist(filesLocaton, FileFactory.getFileType(filesLocaton))) {
                return 0;
            }
            fileChannel = new DataInputStream(
                    FileFactory.getDataInputStream(filesLocaton, FileFactory.getFileType(filesLocaton), 10240));
            return fileChannel.readInt();

        } catch (IOException e) {
            //            e.printStackTrace();
            LOGGER.error(e, e.getMessage());
        } finally {
            CarbonUtil.closeStreams(fileChannel);
        }
        return 0;
    }

    public static int getMaxValueFromLevelFile(String filesLocaton) {
        if (null == filesLocaton) {
            return 0;
        }
        DataInputStream fileChannel = null;
        try {
            if (!FileFactory.isFileExist(filesLocaton, FileFactory.getFileType(filesLocaton))) {
                return 0;
            }
            fileChannel = new DataInputStream(
                    FileFactory.getDataInputStream(filesLocaton, FileFactory.getFileType(filesLocaton), 10240));
            CarbonFile memberFile = FileFactory.getCarbonFile(filesLocaton, FileFactory.getFileType(filesLocaton));
            long size = memberFile.getSize() - 4;
            long skipSize = size;
            long actualSkipSize = 0;
            while (actualSkipSize != size) {
                actualSkipSize += fileChannel.skip(skipSize);
                skipSize = skipSize - actualSkipSize;
            }
            LOGGER.debug("Bytes skipped " + skipSize);
            int maxVal = fileChannel.readInt();
            return maxVal;

        } catch (IOException e) {
            //            e.printStackTrace();
            LOGGER.error(e, e.getMessage());
        } finally {
            CarbonUtil.closeStreams(fileChannel);
        }
        return 0;
    }

    public static int[] getGlobalSurrogateMapping(String filesLocaton) {
        int[] globalMapping = new int[0];
        if (null == filesLocaton) {
            return globalMapping;
        }
        DataInputStream fileChannel = null;
        try {
            if (!FileFactory.isFileExist(filesLocaton, FileFactory.getFileType(filesLocaton))) {
                return null;
            }
            fileChannel = FileFactory.getDataInputStream(filesLocaton, FileFactory.getFileType(filesLocaton));
            fileChannel.readInt();
            int minValue = fileChannel.readInt();
            int numberOfEntries = fileChannel.readInt();
            globalMapping = new int[numberOfEntries];
            int counter = 0;
            int index = 0;
            while (counter < numberOfEntries) {
                index = fileChannel.readInt();
                globalMapping[index - minValue] = fileChannel.readInt();
                counter++;
            }
        } catch (IOException e) {
            //            e.printStackTrace();
            LOGGER.error(e, e.getMessage());
        } finally {
            CarbonUtil.closeStreams(fileChannel);
        }
        return globalMapping;
    }

    public static Map<Integer, Integer> getGlobalSurrogateMappingMapBased(String filesLocaton) {
        if (null == filesLocaton) {
            return null;
        }
        DataInputStream fileChannel = null;
        Map<Integer, Integer> map = new HashMap<Integer, Integer>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
        try {
            if (!FileFactory.isFileExist(filesLocaton, FileFactory.getFileType(filesLocaton))) {
                return null;
            }
            fileChannel = FileFactory.getDataInputStream(filesLocaton, FileFactory.getFileType(filesLocaton));
            fileChannel.readInt();
            int numberOfEntries = fileChannel.readInt();
            int counter = 0;
            while (counter < numberOfEntries) {
                map.put(fileChannel.readInt(), fileChannel.readInt());
                counter++;
            }
        } catch (IOException e) {
            //            e.printStackTrace();
            LOGGER.error(e, e.getMessage());
        } finally {
            CarbonUtil.closeStreams(fileChannel);
        }
        return map;
    }

    /**
     *
     * @param memberFile
     * @param inProgressLoadFolder
     * @return
     * @throws KettleException
     *
     */
    //    private static File decryptEncyptedFile(String memberFile)
    //    {
    //        String decryptedFilePath = memberFile + CarbonCommonConstants.FILE_INPROGRESS_STATUS;
    //
    //        try
    //        {
    //            SimpleFileEncryptor.decryptFile(memberFile , decryptedFilePath);
    //        }
    //        catch(CipherException e)
    //        {
    //            LOGGER.error(CarbonEngineLogEvent.UNIBI_CARBONENGINE_MSG, "Error while decrypting
    // File");
    //        }
    //        catch(IOException e)
    //        {
    //            LOGGER.error(
    //                    CarbonEngineLogEvent.UNIBI_CARBONENGINE_MSG,
    //                    e, "Not able to encrypt File");
    //        }
    //
    //        return new File(decryptedFilePath);
    //    }

    /**
     * @param nameColumnIndex
     * @param filename
     * @param members
     * @throws IOException
     * @author A00902732
     * </br>     Reads the member file
     * </br>
     * Member file structure
     * </br>           |1|2|3|4|5|6|7|8....
     * </br>           |1|2|3|4|5|6|....
     * </br>
     * </br>            1#totalRecoredLength - exclusive - 4 bytes
     * </br>            2#valueLength - 4 bytes
     * </br>            3#Value - depends on previous length
     * </br>            4#surrogateKey - 4 bytes
     * </br>            5#property1Length - 4 bytes
     * </br>            6#property1Value - depends on prev length
     * </br>            7#property2Length - 4 bytes
     * </br>            8#property2Value - depends on prev length
     * </br>            ...
     * Reads accordingly
     */
    public static Member[][] processMemberFile(String filename, String dataType)

    {
        long startTime = System.currentTimeMillis();
        Member[][] members = null;
        // create an object of FileOutputStream
        DataInputStream fileChannel = null;
        try {
            try {
                if (!FileFactory.isFileExist(filename, FileFactory.getFileType(filename))) {
                    return members;
                }
            } catch (IOException e) {
                return members;
            }
            //
            fileChannel = FileFactory.getDataInputStream(filename, FileFactory.getFileType(filename));
            FileType fileType = FileFactory.getFileType(filename);
            CarbonFile memberFile = FileFactory.getCarbonFile(filename, fileType);
            members = populateMemberCache(fileChannel, memberFile, filename, dataType);
        } catch (FileNotFoundException f) {
            LOGGER.error("@@@@@@  Member file is missing @@@@@@ : " + filename);
        } catch (IOException e) {
            LOGGER.error("@@@@@@  Error while reading Member the file @@@@@@ : " + filename);
        } finally {
            CarbonUtil.closeStreams(fileChannel);
        }

        LOGGER.debug(
                "Time taken to process file " + filename + " is : " + (System.currentTimeMillis() - startTime));
        return members;
    }

    public static void processMemberFileNewImpl(int nameColumnIndex, String filename, Int2ObjectMap<Member> members,
            String dataType)

    {
        long startTime = System.currentTimeMillis();
        // Coverity Fix add null check
        if (null == filename || null == members) {
            return;
        }
        // create an object of FileOutputStream
        DataInputStream fileChannel = null;
        try {
            //
            try {
                if (!FileFactory.isFileExist(filename, FileFactory.getFileType(filename))) {
                    return;
                }
            } catch (IOException e) {
                return;
            }
            FileType fileType = FileFactory.getFileType(filename);
            CarbonFile memberFile = FileFactory.getCarbonFile(filename, fileType);

            fileChannel = FileFactory.getDataInputStream(filename, FileFactory.getFileType(filename));
            populateMemberCache(fileChannel, memberFile, filename, dataType);
        } catch (FileNotFoundException f) {
            LOGGER.error("@@@@@@  Member file is missing @@@@@@ : " + filename);
        } catch (IOException e) {
            LOGGER.error("@@@@@@  Error while reading Member the file @@@@@@ : " + filename);
        } finally {
            CarbonUtil.closeStreams(fileChannel);
        }

        LOGGER.debug(
                "Time taken to process file " + filename + " is : " + (System.currentTimeMillis() - startTime));

    }

    private static Member[][] populateMemberCache(DataInputStream fileChannel, CarbonFile memberFile,
            String fileName, String dataType) throws IOException {
        // ByteBuffer toltalLength, memberLength, surrogateKey, bf3;
        // subtracted 4 as last 4 bytes will have the max value for no of
        // records
        long currPositionIndex = 0;
        long size = memberFile.getSize() - 4;
        long skipSize = size;
        long actualSkipSize = 0;
        while (actualSkipSize != size) {
            actualSkipSize += fileChannel.skip(skipSize);
            skipSize = skipSize - actualSkipSize;
        }
        //        LOGGER.debug(CarbonEngineLogEvent.UNIBI_CARBONENGINE_MSG, "Bytes skipped " +
        // skipSize);
        int maxVal = fileChannel.readInt();
        CarbonUtil.closeStreams(fileChannel);
        fileChannel = FileFactory.getDataInputStream(fileName, FileFactory.getFileType(fileName));
        // CHECKSTYLE:OFF Approval No:Approval-V1R2C10_005
        ByteBuffer buffer = ByteBuffer.allocate((int) size);
        // CHECKSTYLE:OFF
        fileChannel.readFully(buffer.array());
        int minVal = buffer.getInt();
        int totalArraySize = maxVal - minVal + 1;
        Member[][] surogateKeyArrays = null;
        if (totalArraySize > CarbonCommonConstants.LEVEL_ARRAY_SIZE) {
            int div = totalArraySize / CarbonCommonConstants.LEVEL_ARRAY_SIZE;
            int rem = totalArraySize % CarbonCommonConstants.LEVEL_ARRAY_SIZE;
            if (rem > 0) {
                div++;
            }
            surogateKeyArrays = new Member[div][];

            for (int i = 0; i < div - 1; i++) {
                surogateKeyArrays[i] = new Member[CarbonCommonConstants.LEVEL_ARRAY_SIZE];
            }

            if (rem > 0) {
                surogateKeyArrays[surogateKeyArrays.length - 1] = new Member[rem];
            } else {
                surogateKeyArrays[surogateKeyArrays.length
                        - 1] = new Member[CarbonCommonConstants.LEVEL_ARRAY_SIZE];
            }
        } else {
            surogateKeyArrays = new Member[1][totalArraySize];
        }
        //        Member[] surogateKeyArrays = new Member[maxVal-minVal+1];
        //        int surrogateKeyIndex = minVal;
        currPositionIndex += 4;
        //
        int current = 0;
        // CHECKSTYLE:OFF Approval No:Approval-V1R2C10_005
        boolean enableEncoding = Boolean
                .valueOf(CarbonProperties.getInstance().getProperty(CarbonCommonConstants.ENABLE_BASE64_ENCODING,
                        CarbonCommonConstants.ENABLE_BASE64_ENCODING_DEFAULT));
        // CHECKSTYLE:ON
        int index = 0;
        int prvArrayIndex = 0;
        while (currPositionIndex < size) {
            int len = buffer.getInt();
            // CHECKSTYLE:OFF Approval No:Approval-V1R2C10_005
            // CHECKSTYLE:ON
            currPositionIndex += 4;
            byte[] rowBytes = new byte[len];
            buffer.get(rowBytes);
            currPositionIndex += len;
            // No:Approval-361
            if (enableEncoding) {
                rowBytes = Base64.decodeBase64(rowBytes);
            }
            surogateKeyArrays[current / CarbonCommonConstants.LEVEL_ARRAY_SIZE][index] = new Member(rowBytes);
            current++;
            if (current / CarbonCommonConstants.LEVEL_ARRAY_SIZE > prvArrayIndex) {
                prvArrayIndex++;
                index = 0;
            } else {
                index++;
            }
        }
        return surogateKeyArrays;
    }

    /**
     * Below method is responsible for reading the sort index and sort reverse index
     *
     * @param levelFileName
     * @return List<int[][]>
     * @throws IOException
     */
    public static List<int[][]> getLevelSortOrderAndReverseIndex(String levelFileName) throws IOException {
        if (!FileFactory.isFileExist(levelFileName + CarbonCommonConstants.LEVEL_SORT_INDEX_FILE_EXT,
                FileFactory.getFileType(levelFileName + CarbonCommonConstants.LEVEL_SORT_INDEX_FILE_EXT))) {
            return null;
        }
        DataInputStream dataInputStream = null;
        List<int[][]> sortIndexAndReverseIndexArray = new ArrayList<int[][]>(
                CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
        long size = 0;
        ByteBuffer buffer = null;
        try {
            dataInputStream = FileFactory.getDataInputStream(
                    levelFileName + CarbonCommonConstants.LEVEL_SORT_INDEX_FILE_EXT,
                    FileFactory.getFileType(levelFileName));
            size = FileFactory.getCarbonFile(levelFileName + CarbonCommonConstants.LEVEL_SORT_INDEX_FILE_EXT,
                    FileFactory.getFileType(levelFileName)).getSize();
            // CHECKSTYLE:OFF Approval No:Approval-V1R2C10_005
            buffer = ByteBuffer.allocate((int) size);
            // CHECKSTYLE:ON
            dataInputStream.readFully(buffer.array());
            sortIndexAndReverseIndexArray.add(getIndexArray(buffer));
            sortIndexAndReverseIndexArray.add(getIndexArray(buffer));
        } catch (IOException e) {
            LOGGER.error(e);
            throw e;
        } finally {
            CarbonUtil.closeStreams(dataInputStream);
        }
        return sortIndexAndReverseIndexArray;
    }

    private static int[][] getIndexArray(ByteBuffer buffer) {
        int arraySize = buffer.getInt();
        int[][] sortorderIndexArray = new int[1][arraySize];
        if (arraySize > CarbonCommonConstants.LEVEL_ARRAY_SIZE) {
            int div = arraySize / CarbonCommonConstants.LEVEL_ARRAY_SIZE;
            int rem = arraySize % CarbonCommonConstants.LEVEL_ARRAY_SIZE;
            if (rem > 0) {
                div++;
            }
            sortorderIndexArray = new int[div][];
            for (int i = 0; i < div - 1; i++) {
                sortorderIndexArray[i] = new int[CarbonCommonConstants.LEVEL_ARRAY_SIZE];
            }

            if (rem > 0) {
                sortorderIndexArray[sortorderIndexArray.length - 1] = new int[rem];
            } else {
                sortorderIndexArray[sortorderIndexArray.length
                        - 1] = new int[CarbonCommonConstants.LEVEL_ARRAY_SIZE];
            }
        }
        int index = 0;
        int prvArrayIndex = 0;
        int current = 0;
        for (int i = 0; i < arraySize; i++) {
            sortorderIndexArray[current / CarbonCommonConstants.LEVEL_ARRAY_SIZE][index] = buffer.getInt();
            current++;
            if (current / CarbonCommonConstants.LEVEL_ARRAY_SIZE > prvArrayIndex) {
                prvArrayIndex++;
                index = 0;
            } else {
                index++;
            }
        }
        return sortorderIndexArray;
    }

    /**
     * Read hierarchies from file. Hierarchies contain duplicate entries.
     * Hierarchy: Filename= HierachyName Content = byte content
     *
     * @param hName
     * @param filesLocaton
     * @param keyLength
     * @param keyGen
     * @return
     */
    public static List<long[]> getHierarchiesList(String filesLocaton, int keyLength, KeyGenerator keyGen) {
        LOGGER.debug("Reading hierarchies from location: " + filesLocaton);

        // hierarchies list , this will hold hierarchy surrogate key
        List<long[]> hierarchies = new ArrayList<long[]>(LIST_INTIAL_CAPECITY);

        // set is used to remove duplicate entries
        Set<ArrayWrapper> wrapHiers = new HashSet<ArrayWrapper>(SET_INTIAL_CAPECITY);

        FileInputStream reader = null;
        try {
            reader = new FileInputStream(filesLocaton);
            // get key length
            byte[] line = new byte[keyLength];
            while (reader.read(line) == keyLength) {
                // add to set
                wrapHiers.add(new ArrayWrapper(keyGen.getKeyArray(line)));
            }
        } catch (IOException e) {
            LOGGER.error("Error while reading the file: " + filesLocaton);
        } finally {
            try {
                if (reader != null) {
                    reader.close();
                }
            } catch (IOException e) {
                LOGGER.error("Error while closing the file stream for file " + filesLocaton);
            }
        }

        // copy all the hierarchies from set to list
        for (ArrayWrapper wrapHier : wrapHiers) {
            hierarchies.add(wrapHier.getData());
        }
        return hierarchies;
    }

    /**
     * This method is used to check whether any exclude filter is present in
     * constraints
     *
     * @param constraints Dimension and its filter info
     * @return true if exclude filter is present
     */
    public static boolean checkAnyExcludeExists(Map<Dimension, CarbonFilterInfo> constraints) {
        for (Map.Entry<Dimension, CarbonFilterInfo> entry : constraints.entrySet()) {
            // check if filter size is greater than zero, if it is more than
            // zero than include filter is present
            if (entry.getValue().getExcludedMembers().size() > 0) {
                return true;
            }
        }
        return false;
    }

    /**
     * This method is used to check whether any include filter is present in
     * constraints
     *
     * @param constraints Dimension and its filter info
     * @return true if include filter is present
     */
    public static boolean checkAnyIncludeExists(Map<Dimension, CarbonFilterInfo> constraints) {
        for (Map.Entry<Dimension, CarbonFilterInfo> entry : constraints.entrySet()) {
            // check if filter size is greater than zero, if it is more than
            // zero than include filter is present
            if (entry.getValue().getIncludedMembers().size() > 0) {
                return true;
            }
        }
        return false;
    }

    /**
     * This method is used to check whether any include filter is present in
     * constraints
     *
     * @param constraints Dimension and its filter info
     * @return true if include filter is present
     */
    public static boolean checkAnyIncludeOrExists(Map<Dimension, CarbonFilterInfo> constraints) {
        for (Map.Entry<Dimension, CarbonFilterInfo> entry : constraints.entrySet()) {
            // check if filter size is greater than zero, if it is more than
            // zero than include filter is present
            if (entry.getValue().getIncludedOrMembers() != null
                    && entry.getValue().getIncludedOrMembers().size() > 0) {
                return true;
            }
        }
        return false;
    }

    /**
     * This method will return the size of a given file
     *
     * @param fileName
     * @return
     */
    public static long getMemberFileSize(String fileName) {
        if (isFileExists(fileName)) {
            FileType fileType = FileFactory.getFileType(fileName);
            CarbonFile memberFile = FileFactory.getCarbonFile(fileName, fileType);
            return memberFile.getSize();
        }
        return 0;
    }

    /**
     * @param fileName
     * @param fileType
     */
    public static boolean isFileExists(String fileName) {
        try {
            FileType fileType = FileFactory.getFileType(fileName);
            if (FileFactory.isFileExist(fileName, fileType)) {
                return true;
            }
        } catch (IOException e) {
            LOGGER.error("@@@@@@  Member file not found for size to be calculated @@@@@@ : " + fileName);
        }
        return false;
    }

    /**
     * This method will return the actual level name based on the dimension
     *
     * @param schema
     * @param dimension
     * @return
     */
    public static String getLevelActualName(CarbonDef.Schema schema, CarbonDef.CubeDimension dimension) {
        String levelActualName = null;
        CarbonDef.Hierarchy[] extractHierarchies = CarbonSchemaReader.extractHierarchies(schema, dimension);
        if (null != extractHierarchies) {
            for (CarbonDef.Hierarchy hierarchy : extractHierarchies) {
                for (CarbonDef.Level level : hierarchy.levels) {
                    levelActualName = level.column;
                    break;
                }
            }
        }
        return levelActualName;
    }

}