com.alibaba.wasp.meta.FMetaStore.java Source code

Java tutorial

Introduction

Here is the source code for com.alibaba.wasp.meta.FMetaStore.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.alibaba.wasp.meta;

import com.alibaba.wasp.EntityGroupInfo;
import com.alibaba.wasp.EntityGroupLocation;
import com.alibaba.wasp.FConstants;
import com.alibaba.wasp.MetaException;
import com.alibaba.wasp.ServerName;
import com.alibaba.wasp.coprocessor.WaspAggregateImplementation;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.ConnectionUtils;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.Set;
import java.util.TreeMap;

/**
 * implement FMetaServices
 **/
class FMetaStore extends FMetaServices {

    public static final Log LOG = LogFactory.getLog(FMetaStore.class);

    protected FMetaServicesImplWithoutRetry fMetaServices;
    private int maxRetries;
    private long pause;

    public FMetaStore(Configuration conf) throws MetaException {
        this.setConf(conf);
        fMetaServices = new FMetaServicesImplWithoutRetry(conf);
        maxRetries = conf.getInt(FConstants.WASP_FMETA_RETRIES_NUMBER,
                FConstants.DEFAULT_WASP_FMETA_RETRIES_NUMBER);
        pause = conf.getLong(HConstants.HBASE_CLIENT_PAUSE, HConstants.DEFAULT_HBASE_CLIENT_PAUSE);
    }

    /**
     * Use HTable to connect to the meta table, will use the HTable to
     * create\drop\alter\get Table and so on.
     */
    @Override
    public boolean connect() {
        return fMetaServices.connect();
    }

    @Override
    public void close() {
        fMetaServices.close();
    }

    @Override
    public HTableDescriptor[] listStorageTables() throws MetaException {
        return fMetaServices.listStorageTables();
    }

    @Override
    public HTableDescriptor getStorageTableDesc(String storageTable) throws MetaException {
        return fMetaServices.getStorageTableDesc(storageTable);
    }

    @Override
    public void deleteStorageTables(List<HTableDescriptor> deleteEntityTables) throws MetaException {
        fMetaServices.deleteStorageTables(deleteEntityTables);
    }

    @Override
    public void deleteStorageTable(String deleteStorageTable) throws MetaException {
        fMetaServices.deleteStorageTable(deleteStorageTable);
    }

    @Override
    public boolean storageTableExists(String deleteStorageTable) throws MetaException {
        return fMetaServices.storageTableExists(deleteStorageTable);
    }

    @Override
    public void createTable(FTable tbl) throws MetaException {
        fMetaServices.createTable(tbl);
        try {
            // This method will be concurrent called
            // current we thought this is might be a right place to create the hbase
            // table
            HTableDescriptor desc = getStorageTableDesc(tbl);
            //      if(!desc.hasCoprocessor(AggregateImplementation.class.getName())) {
            //        desc.addCoprocessor(AggregateImplementation.class.getName());
            //      }
            if (!desc.hasCoprocessor(WaspAggregateImplementation.class.getName())) {
                desc.addCoprocessor(WaspAggregateImplementation.class.getName());
            }
            createStorageTable(desc);
        } catch (IOException e) {
            LOG.error("Failed to create table.", e);
            throw new MetaException(e);
        }
    }

    @Override
    public void dropTable(String tableName) throws MetaException {
        LinkedHashMap<String, Index> indexs = fMetaServices.getAllIndex(tableName);
        if (indexs != null) {
            for (Index index : indexs.values()) {
                String htable = StorageTableNameBuilder.buildIndexTableName(index);
                deleteTable(htable);
            }
        }
        String htablename = StorageTableNameBuilder.buildEntityTableName(tableName);
        deleteTable(htablename);
        fMetaServices.dropTable(tableName);
    }

    private void deleteTable(String tableName) throws MetaException {
        try {
            // This method will be concurrent called
            // current we thought this is might be a right place to create the hbase
            // table
            HBaseAdmin admin = new HBaseAdmin(getConf());
            if (!admin.tableExists(tableName)) {
                LOG.warn(tableName + " have been already deleted.");
                admin.close();
                return;
            }
            admin.disableTable(tableName);
            try {
                admin.deleteTable(tableName);
            } finally {
                admin.close();
            }
        } catch (MasterNotRunningException e) {
            LOG.error("Failed to delete table.", e);
            throw new MetaException(e);
        } catch (ZooKeeperConnectionException e) {
            LOG.error("Failed to delete table.", e);
            throw new MetaException(e);
        } catch (IOException e) {
            LOG.error("Failed to delete table.", e);
            throw new MetaException(e);
        }
    }

    @Override
    public void alterTable(String tableName, FTable newFTable) throws MetaException {
        fMetaServices.alterTable(tableName, newFTable);
    }

    @Override
    public FTable getTable(String tableName) throws MetaException {
        return fMetaServices.getTable(tableName);
    }

    @Override
    public List<FTable> getTables(String regex) throws MetaException {
        return fMetaServices.getTables(regex);
    }

    @Override
    public List<FTable> getAllTables() throws MetaException {
        return fMetaServices.getAllTables();
    }

    @Override
    public List<FTable> getChildTables(String tableName) throws MetaException {
        return fMetaServices.getChildTables(tableName);
    }

    @Override
    public boolean tableExists(String tableName) throws MetaException {
        return fMetaServices.tableExists(tableName);
    }

    // ///////////////////////////////////////////////////
    // FTable operation
    // ///////////////////////////////////////////////////

    @Override
    public void addIndex(String tableName, Index index) throws MetaException {
        fMetaServices.addIndex(tableName, index);
        try {
            // This method will be concurrent called
            // current we thought this is might be a right place to create the hbase
            // table
            HBaseAdmin admin = new HBaseAdmin(getConf());
            try {
                String htable = StorageTableNameBuilder.buildIndexTableName(index);
                HTableDescriptor desc = new HTableDescriptor(htable);
                HColumnDescriptor hcolumn = new HColumnDescriptor(FConstants.INDEX_STORING_FAMILY_STR);
                hcolumn.setBloomFilterType(BloomType.ROW);
                hcolumn.setCompressionType(Algorithm.GZ);
                desc.addFamily(hcolumn);
                admin.createTable(desc);
            } finally {
                admin.close();
            }
        } catch (MasterNotRunningException e) {
            LOG.error("Failed to create index table.", e);
            throw new MetaException(e);
        } catch (ZooKeeperConnectionException e) {
            LOG.error("Failed to create index table.", e);
            throw new MetaException(e);
        } catch (IOException e) {
            LOG.error("Failed to create index table.", e);
            throw new MetaException(e);
        }
    }

    @Override
    public void deleteIndex(String tableName, Index index) throws MetaException {
        fMetaServices.deleteIndex(tableName, index);
        String htable = StorageTableNameBuilder.buildIndexTableName(index);
        deleteTable(htable);
    }

    @Override
    public LinkedHashMap<String, Index> getAllIndex(String tableName) throws MetaException {
        return fMetaServices.getAllIndex(tableName);
    }

    @Override
    public Index getIndex(String tableName, String indexName) throws MetaException {
        return fMetaServices.getIndex(tableName, indexName);
    }

    // ///////////////////////////////////////////////////
    // EntityGroup operation
    // ///////////////////////////////////////////////////
    @Override
    public void addEntityGroup(EntityGroupInfo entityGroupInfo) throws MetaException {
        fMetaServices.addEntityGroup(entityGroupInfo);
    }

    @Override
    public void addEntityGroup(List<EntityGroupInfo> entityGroupInfos) throws MetaException {
        fMetaServices.addEntityGroup(entityGroupInfos);
    }

    @Override
    public boolean exists(EntityGroupInfo entityGroupInfo) throws MetaException {
        return fMetaServices.exists(entityGroupInfo);
    }

    @Override
    public void deleteEntityGroup(EntityGroupInfo entityGroupInfo) throws MetaException {
        fMetaServices.deleteEntityGroup(entityGroupInfo);
    }

    @Override
    public void deleteEntityGroups(String tableName) throws MetaException {
        fMetaServices.deleteEntityGroups(tableName);
    }

    @Override
    public List<EntityGroupInfo> getTableEntityGroups(String tableName) throws MetaException {
        FTable root = fMetaServices.getRootTable(tableName);
        byte[] rootName = Bytes.toBytes(root.getTableName());
        // use scan
        for (int tries = 0; tries < maxRetries; ++tries) {
            try {
                return fMetaServices.getTableEntityGroups(rootName);
            } catch (MetaException me) {
                if (tries == maxRetries - 1) {
                    throw new MetaException(" RetriesExhaustedException ", me);
                }
                LOG.info("RetryCount " + tries, me);
            }
            try {
                Thread.sleep(ConnectionUtils.getPauseTime(pause, tries));
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                throw new MetaException("Giving up after tries=" + tries, e);
            }
        }
        return new ArrayList<EntityGroupInfo>(0);
    }

    @Override
    public List<EntityGroupInfo> getTableEntityGroups(final byte[] tableByte) throws MetaException {
        FTable root = fMetaServices.getRootTable(tableByte);
        byte[] rootName = Bytes.toBytes(root.getTableName());
        // use scan
        for (int tries = 0; tries < maxRetries; ++tries) {
            try {
                return fMetaServices.getTableEntityGroups(rootName);
            } catch (MetaException me) {
                if (tries == maxRetries - 1) {
                    throw new MetaException(" RetriesExhaustedException ", me);
                }
                LOG.info("RetryCount " + tries, me);
            }
            try {
                Thread.sleep(ConnectionUtils.getPauseTime(pause, tries));
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                throw new MetaException("Giving up after tries=" + tries, e);
            }
        }
        return new ArrayList<EntityGroupInfo>(0);
    }

    @Override
    public void modifyEntityGroupInfo(EntityGroupInfo entityGroupInfo) throws MetaException {
        fMetaServices.modifyEntityGroupInfo(entityGroupInfo);
    }

    @Override
    public EntityGroupInfo getEntityGroupInfo(EntityGroupInfo entityGroupInfo) throws MetaException {
        return fMetaServices.getEntityGroupInfo(entityGroupInfo);
    }

    @Override
    public ServerName getEntityGroupLocation(EntityGroupInfo entityGroupInfo) throws MetaException {
        return fMetaServices.getEntityGroupLocation(entityGroupInfo);
    }

    @Override
    public Pair<EntityGroupInfo, ServerName> getEntityGroupAndLocation(byte[] entityGroupName)
            throws MetaException {
        return fMetaServices.getEntityGroupAndLocation(entityGroupName);
    }

    @Override
    public Map<EntityGroupInfo, ServerName> fullScan(final Set<String> disabledTables,
            final boolean excludeOfflinedSplitParents) throws MetaException {
        // use scan
        for (int tries = 0; tries < maxRetries; ++tries) {
            try {
                return fMetaServices.fullScan(disabledTables, excludeOfflinedSplitParents);
            } catch (MetaException me) {
                if (tries == maxRetries - 1) {
                    throw new MetaException(" RetriesExhaustedException ", me);
                }
                LOG.info("RetryCount " + tries, me);
            }
            try {
                Thread.sleep(ConnectionUtils.getPauseTime(pause, tries));
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                throw new MetaException("Giving up after tries=" + tries, e);
            }
        }
        return new HashMap<EntityGroupInfo, ServerName>(0);
    }

    @Override
    public List<Result> fullScan() throws MetaException {
        // use scan
        for (int tries = 0; tries < maxRetries; ++tries) {
            try {
                return fMetaServices.fullScan();
            } catch (MetaException me) {
                if (tries == maxRetries - 1) {
                    throw new MetaException("RetriesExhaustedException ", me);
                }
                LOG.info("RetryCount " + tries, me);
            }
            try {
                Thread.sleep(ConnectionUtils.getPauseTime(pause, tries));
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                throw new MetaException("Giving up after tries=" + tries, e);
            }
        }
        return new ArrayList<Result>(0);
    }

    @Override
    public Map<EntityGroupInfo, Result> getOfflineSplitParents() throws MetaException {
        // use scan
        for (int tries = 0; tries < maxRetries; ++tries) {
            try {
                return fMetaServices.getOfflineSplitParents();
            } catch (MetaException me) {
                if (tries == maxRetries - 1) {
                    throw new MetaException(" RetriesExhaustedException ", me);
                }
                LOG.info("RetryCount " + tries, me);
            }
            try {
                Thread.sleep(ConnectionUtils.getPauseTime(pause, tries));
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                throw new MetaException("Giving up after tries=" + tries, e);
            }
        }

        return new HashMap<EntityGroupInfo, Result>(0);
    }

    @Override
    public NavigableMap<EntityGroupInfo, Result> getServerUserEntityGroups(final ServerName serverName)
            throws MetaException {
        // use scan
        for (int tries = 0; tries < maxRetries; ++tries) {
            try {
                return fMetaServices.getServerUserEntityGroups(serverName);
            } catch (MetaException me) {
                if (tries == maxRetries - 1) {
                    throw new MetaException(" RetriesExhaustedException ", me);
                }
                LOG.info("RetryCount " + tries, me);
            }
            try {
                Thread.sleep(ConnectionUtils.getPauseTime(pause, tries));
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                throw new MetaException("Giving up after tries=" + tries, e);
            }
        }
        return new TreeMap<EntityGroupInfo, Result>();
    }

    @Override
    public List<EntityGroupInfo> getAllEntityGroupInfos() throws MetaException {
        return fMetaServices.getAllEntityGroupInfos();
    }

    @Override
    public void fullScan(final FMetaVisitor visitor) throws MetaException {
        // use scan
        fMetaServices.fullScan(visitor);
    }

    @Override
    public void fullScan(final FMetaVisitor visitor, final byte[] startrow, final byte[] endrow)
            throws MetaException {
        // use scan
        fMetaServices.fullScan(visitor, startrow, endrow);
    }

    @Override
    public void fullScan(final FMetaVisitor visitor, final byte[] startrow, final byte[] endrow,
            FilterList allFilters) throws MetaException {
        // use scan
        fMetaServices.fullScan(visitor, startrow, endrow, allFilters);
    }

    @Override
    public void updateEntityGroupLocation(EntityGroupInfo entityGroupInfo, ServerName sn) throws MetaException {
        fMetaServices.updateEntityGroupLocation(entityGroupInfo, sn);
    }

    @Override
    public EntityGroupLocation scanEntityGroupLocation(final byte[] tableName, final byte[] row)
            throws MetaException {
        // use scan
        for (int tries = 0; tries < maxRetries; ++tries) {
            try {
                return fMetaServices.scanEntityGroupLocation(tableName, row);
            } catch (MetaException me) {
                if (tries == maxRetries - 1) {
                    LOG.error("RetriesExhaustedException", me);
                    throw new MetaException("RetriesExhaustedException ", me);
                }
                LOG.info("RetryCount " + tries, me);
            }
            try {
                Thread.sleep(ConnectionUtils.getPauseTime(pause, tries));
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                throw new MetaException("Giving up after tries=" + tries, e);
            }
        }
        return null;
    }

    @Override
    public List<EntityGroupLocation> getEntityGroupLocations(final byte[] tableName) throws MetaException {
        FTable root = fMetaServices.getRootTable(tableName);
        byte[] rootName = Bytes.toBytes(root.getTableName());
        // use scan
        for (int tries = 0; tries < maxRetries; ++tries) {
            try {
                return fMetaServices.getEntityGroupLocations(rootName);
            } catch (MetaException me) {
                if (tries == maxRetries - 1) {
                    throw new MetaException(" RetriesExhaustedException ", me);
                }
                LOG.info(" RetryCount " + tries, me);
            }
            try {
                Thread.sleep(ConnectionUtils.getPauseTime(pause, tries));
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                throw new MetaException("Giving up after tries=" + tries, e);
            }
        }
        return new ArrayList<EntityGroupLocation>(0);
    }

    /**
     * @param tableName
     * @return Return list of EntityGroupInfos and server addresses.
     * @throws java.io.IOException
     * @throws InterruptedException
     */
    @Override
    public List<Pair<EntityGroupInfo, ServerName>> getTableEntityGroupsAndLocations(final byte[] tableName,
            final boolean excludeOfflinedSplitParents) throws MetaException {
        FTable root = fMetaServices.getRootTable(tableName);
        byte[] rootName = Bytes.toBytes(root.getTableName());
        // use scan
        for (int tries = 0; tries < maxRetries; ++tries) {
            try {
                return fMetaServices.getTableEntityGroupsAndLocations(rootName, excludeOfflinedSplitParents);
            } catch (MetaException me) {
                if (tries == maxRetries - 1) {
                    throw new MetaException(" RetriesExhaustedException ", me);
                }
                LOG.info(" RetryCount " + tries, me);
            }
            try {
                Thread.sleep(ConnectionUtils.getPauseTime(pause, tries));
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                throw new MetaException("Giving up after tries=" + tries, e);
            }
        }
        return new ArrayList<Pair<EntityGroupInfo, ServerName>>(0);
    }

    public void offlineParentInMeta(EntityGroupInfo parent, final EntityGroupInfo a, final EntityGroupInfo b)
            throws MetaException {
        fMetaServices.offlineParentInMeta(parent, a, b);
    }

    /**
     * @param entityGroupInfo
     * @param sn
     * @throws com.alibaba.wasp.MetaException
     */
    @Override
    public void addDaughter(final EntityGroupInfo entityGroupInfo, final ServerName sn) throws MetaException {
        fMetaServices.addDaughter(entityGroupInfo, sn);
    }

    /**
     * Generates and returns a Put containing the EntityGroupInfo into for the
     * catalog table
     */
    public static Put makePutFromEntityGroupInfo(EntityGroupInfo entityGroupInfo) {
        return FMetaServicesImplWithoutRetry.makePutFromEntityGroupInfo(entityGroupInfo);
    }

    /**
     * Deletes daughters references in offlined split parent.
     *
     * @param parent
     *          Parent row we're to remove daughter reference from
     * @throws com.alibaba.wasp.MetaException
     */
    public void deleteDaughtersReferencesInParent(final EntityGroupInfo parent) throws MetaException {
        fMetaServices.deleteDaughtersReferencesInParent(parent);
    }

    public boolean isTableAvailable(final byte[] tableName) throws IOException {
        return fMetaServices.isTableAvailable(tableName);
    }

    public void updateLocation(Configuration conf, EntityGroupInfo entityGroupInfo,
            ServerName serverNameFromMasterPOV) throws MetaException {
        fMetaServices.updateLocation(conf, entityGroupInfo, serverNameFromMasterPOV);
    }

    /**
     * @see com.alibaba.wasp.meta.FMetaServices#getStorageTableDesc(com.alibaba.wasp.meta.FTable)
     */
    @Override
    public HTableDescriptor getStorageTableDesc(FTable tbl) {
        return fMetaServices.getStorageTableDesc(tbl);
    }

    /**
     * @see com.alibaba.wasp.meta.FMetaServices#createStorageTable(org.apache.hadoop.hbase.HTableDescriptor)
     */
    @Override
    public void createStorageTable(HTableDescriptor desc) throws MetaException {
        fMetaServices.createStorageTable(desc);
    }
}