Example usage for java.util NavigableMap get

List of usage examples for java.util NavigableMap get

  1. HOME
  2. Java
  3. java
  4. java.util.*
  5. NavigableMap
  6. get

Introduction

In this page you can find the example usage for java.util NavigableMap get.

Prototype

V get(Object key);

Source Link

Document

Returns the value to which the specified key is mapped, or null if this map contains no mapping for the key.

Usage

From source file:org.apache.hadoop.hbase.extended.loadbalance.strategies.hotspot.HotSpotLoadBalancer.java

@Override
public List<RegionPlan> balanceCluster(Map<ServerName, List<HRegionInfo>> clusterState) {
    initParameters();/*from w w w  .j a  v  a 2s.c  o m*/
    /**
     * <pre>
     * We atleast need two priority queues 
     * a) It would contain HotSpot regions with their load as the moving criteria (max priority queue)
     * b) Non hot spot region with their loads (min priority queue)
     * 
     * Further we need to iterate over these queues and decrease the load so we 
     * need a data structure to build these queues 
     * and lastly we need to return the Region plan.
     * </pre>
     */

    LOG.debug("#################Came in the new Balancer Code and the cluster status is = " + this.status);
    long startTime = System.currentTimeMillis();
    int numServers = clusterState.size();
    if (numServers == 0) {
        LOG.info("numServers=0 so skipping load balancing");
        return null;

    }

    NavigableMap<HotSpotServerAndLoad, List<HotSpotRegionLoad>> regionServerAndServerLoadMap = new TreeMap<HotSpotServerAndLoad, List<HotSpotRegionLoad>>();
    PriorityQueue<HotSpotServerAndLoad> hotspotRegionServers = new PriorityQueue<HotSpotServerAndLoad>(
            numServers, HotSpotServerAndLoad.DESC_LOAD);
    PriorityQueue<HotSpotServerAndLoad> nonHotspotRegionServers = new PriorityQueue<HotSpotServerAndLoad>(
            numServers, HotSpotServerAndLoad.ASC_LOAD);
    HashBiMap<HRegionInfo, HotSpotRegionLoad> allRegionsLoadBiMap = HashBiMap.create();
    LOG.debug("#################clusterState=" + clusterState);
    double normalisedTotalLoadOfAllRegions = initRegionLoadMapsBasedOnInput(clusterState,
            regionServerAndServerLoadMap, allRegionsLoadBiMap);
    LOG.debug("#################normalisedTotalLoadOfAllRegions=" + normalisedTotalLoadOfAllRegions);
    // Check if we even need to do any load balancing
    double average = normalisedTotalLoadOfAllRegions / numServers; // for
    // logging
    // HBASE-3681 check sloppiness first
    LOG.debug("######################## final regionServerAndServerLoadMap == " + regionServerAndServerLoadMap);
    if (!loadBalancingNeeded(numServers, regionServerAndServerLoadMap, normalisedTotalLoadOfAllRegions,
            average)) {
        // we do not need load balancing
        return null;
    }
    double minLoad = normalisedTotalLoadOfAllRegions / numServers;
    double maxLoad = normalisedTotalLoadOfAllRegions % numServers == 0 ? minLoad : minLoad + 1;
    // as we now have to balance stuff, init PQ's
    LOG.debug(String.format("#################minLoad =%s,maxLoad= %s", minLoad, maxLoad));
    for (Map.Entry<HotSpotServerAndLoad, List<HotSpotRegionLoad>> item : regionServerAndServerLoadMap
            .entrySet()) {
        HotSpotServerAndLoad serverLoad = item.getKey();
        if (serverLoad.isHotSpot()) {

            hotspotRegionServers.add(serverLoad);
        } else {
            if (serverLoad.getLoad() < maxLoad) {
                nonHotspotRegionServers.add(serverLoad);
            }
        }
    }
    // Using to check balance result.
    StringBuilder strBalanceParam = new StringBuilder();
    strBalanceParam.append("Balance parameter: numRegions=").append(normalisedTotalLoadOfAllRegions)
            .append(", numServers=").append(numServers).append(", max=").append(maxLoad).append(", min=")
            .append(minLoad);
    LOG.debug(strBalanceParam.toString());
    List<RegionPlan> regionsToReturn = new ArrayList<RegionPlan>();

    while (hotspotRegionServers.size() > 0 && nonHotspotRegionServers.size() > 0) {
        HotSpotServerAndLoad serverToBalance = hotspotRegionServers.poll();
        LOG.debug(String.format("#################serverToBalance =%s",
                serverToBalance.getServerName().getServerName()));
        // get least loaded not hotspot regions of this server
        List<HotSpotRegionLoad> regionList = regionServerAndServerLoadMap.get(serverToBalance);
        // assume it to be sorted asc.
        if (regionList.size() > 0) {
            HotSpotRegionLoad regionToMove = regionList.remove(0);
            HRegionInfo regionMoveInfo = allRegionsLoadBiMap.inverse().get(regionToMove);

            /*
             * regionMoveInfo can be null in case the load map returns us
             * the root and meta regions along with the movable regions But
             * as the clusterState which is passed to us does not contain
             * these regions we can have a situation where
             * regionServerAndServerLoadMap contains some regions which are
             * not present in the allRegionsLoadBiMap
             */
            if (regionMoveInfo != null && !regionMoveInfo.isMetaRegion() && !regionMoveInfo.isRootRegion()
                    && !regionMoveInfo.isMetaTable() && regionToMove.isRegionHotspot()) {
                LOG.debug(String.format(
                        "#################Came to move the region regionMoveInfo=%s;; regionToMove=%s ",
                        regionMoveInfo, regionToMove));
                // move out.
                HotSpotServerAndLoad destinationServer = nonHotspotRegionServers.poll();

                RegionPlan rpl = new RegionPlan(allRegionsLoadBiMap.inverse().get(regionToMove),
                        serverToBalance.getServerName(), destinationServer.getServerName());
                regionsToReturn.add(rpl);
                serverToBalance.modifyLoad(regionToMove.getLoad());
                destinationServer.modifyLoad(-1 * regionToMove.getLoad());
                // reenter them to list. if they satisfy conditions
                if (serverToBalance.getLoad() > minLoad) {
                    hotspotRegionServers.offer(serverToBalance);
                }
                if (destinationServer.getLoad() < maxLoad) {
                    nonHotspotRegionServers.offer(destinationServer);
                }
            }
        }
    }
    LOG.info("Total Time taken to balance = " + (System.currentTimeMillis() - startTime));
    LOG.info(String.format("#################regionsToReturn=%s ", regionsToReturn));
    return regionsToReturn;

}

From source file:org.apache.hadoop.hbase.HBaseTestCase.java

protected void assertResultEquals(final HRegion region, final byte[] row, final byte[] family,
        final byte[] qualifier, final long timestamp, final byte[] value) throws IOException {
    Get get = new Get(row);
    get.setTimeStamp(timestamp);/* ww  w .  j  ava2  s  . c  o m*/
    Result res = region.get(get);
    NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map = res.getMap();
    byte[] res_value = map.get(family).get(qualifier).get(timestamp);

    if (value == null) {
        assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) + " at timestamp " + timestamp,
                null, res_value);
    } else {
        if (res_value == null) {
            fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) + " at timestamp " + timestamp
                    + "\" was expected to be \"" + Bytes.toStringBinary(value) + " but was null");
        }
        if (res_value != null) {
            assertEquals(
                    Bytes.toString(family) + " " + Bytes.toString(qualifier) + " at timestamp " + timestamp,
                    value, new String(res_value));
        }
    }
}

From source file:org.apache.hadoop.hbase.index.coprocessor.master.IndexMasterObserver.java

private boolean checkRegionInTransition(ObserverContext<MasterCoprocessorEnvironment> ctx, HRegionInfo hri) {
    MasterServices master = ctx.getEnvironment().getMasterServices();
    AssignmentManager am = master.getAssignmentManager();
    boolean isRegionInTransition = false;
    String tableName = hri.getTableNameAsString();
    if (false == IndexUtils.isIndexTable(tableName)) {
        NavigableMap<String, RegionState> regionsInTransition = am.getRegionsInTransition();
        RegionState regionState = regionsInTransition.get(hri.getEncodedName());
        if (regionState != null) {
            isRegionInTransition = true;
        } else {//from w  w  w. ja  v a 2  s .  co m
            String indexTableName = IndexUtils.getIndexTableName(tableName);
            for (Entry<String, RegionState> region : regionsInTransition.entrySet()) {
                HRegionInfo regionInfo = region.getValue().getRegion();
                if (indexTableName.equals(regionInfo.getTableNameAsString())) {
                    if (Bytes.compareTo(hri.getStartKey(), regionInfo.getStartKey()) == 0) {
                        isRegionInTransition = true;
                        break;
                    }
                }
            }
        }
    }
    return isRegionInTransition;
}

From source file:org.apache.hadoop.hbase.master.balancer.LocalityAwareLoadBalancer.java

/**
 * This implements the Locality Aware Load Balancer.
 * Information for the algorithm can be found here: https://issues.apache.org/jira/browse/HBASE-10075
 *
 * @param clusterMap Map of regionservers and their load/region information to
 *                   a list of their most loaded regions
 * @return a list of regions to be moved, including source and destination,
 *         or null if cluster is already balanced
 *//*from w ww.j a  va2 s . co m*/
public List<RegionPlan> balanceCluster(Map<ServerName, List<HRegionInfo>> clusterMap) {
    long startTime = System.currentTimeMillis();

    ClusterLoadState cs = new ClusterLoadState(clusterMap);

    float average = cs.getLoadAverage(); // for logging
    int ceiling = (int) Math.ceil(average * (1 + slop));
    NavigableMap<ServerAndLoad, List<HRegionInfo>> serversByLoad = cs.getServersByLoad();

    if (!this.needsBalance(cs)) {
        /*LOG.info("Skipping load balancing because balanced cluster; " +
                "servers=" + cs.getNumServers() + " " +
                "regions=" + cs.getNumRegions() + " average=" + average + " " +
                "mostloaded=" + serversByLoad.lastKey().getLoad() +
                " leastloaded=" + serversByLoad.firstKey().getLoad());*/
        return null;
    }

    // Additional check for locality aware load balancer as it only considers
    // only max loaded servers
    if (!(cs.getMaxLoad() > ceiling)) {
        return null;
    }

    Cluster cluster = new Cluster(clusterMap, new HashMap<String, Deque<RegionLoad>>(), regionLocationFinder);
    int numRegions = cs.getNumRegions();

    LOG.info(" ####################################################################################");
    LOG.info(" Before Locality-aware Balancing");
    LOG.info(" Average=" + average + " Ceiling=" + ceiling + " slop=" + slop);
    /* for (ServerAndLoad server : serversByLoad.keySet()) {
      LOG.info("---------------" + "Server Name: " + server.getServerName() + "---------------");
      List<HRegionInfo> hRegionInfos = serversByLoad.get(server);
      LOG.info("Number of Regions:" + hRegionInfos.size());
      for (HRegionInfo hRegionInfo : hRegionInfos){
        LOG.info(String.format("Name of Region: %s ", hRegionInfo.getRegionNameAsString()));
        //LOG.info(String.format("Size of Region in number of rows"+(Bytes.toInt(hRegionInfo.getStartKey())-Bytes.toInt(hRegionInfo.getEndKey()))));
        LOG.info("Start Key: " + Bytes.toString(hRegionInfo.getStartKey()));
        LOG.info("End Key: " + Bytes.toString(hRegionInfo.getEndKey()));
      }
      LOG.info("------------------------------------------------------------------------------");
    } */

    // calculate allTableRegionNumber = total number of regions per table.
    Map<Integer, Integer> allTableRegionNumberMap = new HashMap<Integer, Integer>();
    for (int i = 0; i < cluster.numServers; ++i) {
        for (int j = 0; j < cluster.numTables; ++j) {
            if (allTableRegionNumberMap.containsKey(j)) {
                Integer integer = allTableRegionNumberMap.get(j);
                integer = integer + cluster.numRegionsPerServerPerTable[i][j];
                allTableRegionNumberMap.put(j, integer);
            } else {
                allTableRegionNumberMap.put(j, cluster.numRegionsPerServerPerTable[i][j]);
            }
        }
    }

    List<RegionPlan> regionsToReturn = new ArrayList<RegionPlan>();

    for (ServerAndLoad server : serversByLoad.keySet()) {
        List<HRegionInfo> hRegionInfos = serversByLoad.get(server);
        // Check if number of regions on current server is greater than floor.
        // Continue only if number regions is greater than floor.
        if (hRegionInfos.size() <= ceiling) {
            LOG.debug("Number of HRegions <= ceiling (" + hRegionInfos.size() + " <= " + ceiling + ")");
            continue;
        }
        PriorityQueue<RegionServerRegionAffinity> queue = new PriorityQueue<RegionServerRegionAffinity>();
        int numberOfRegionsToMove = hRegionInfos.size() - ceiling;
        double regionAffinityNumber = (1 - hRegionInfos.size() / numRegions) * SERVER_BALANCER_WEIGHT;
        double tableRegionAffinityNumber = 0;
        // Calculate allTableRegionNumber
        for (HRegionInfo hRegionInfo : hRegionInfos) {
            // Do not move metaregion.
            if (hRegionInfo.isMetaRegion()) {
                continue;
            }
            TableName table = hRegionInfo.getTable();
            String tableName = table.getNameAsString();
            int tableIndex = cluster.tablesToIndex.get(tableName);
            int serverIndex = cluster.serversToIndex.get(server.getServerName().getHostAndPort());
            tableRegionAffinityNumber = (1 - cluster.numRegionsPerServerPerTable[serverIndex][tableIndex]
                    / allTableRegionNumberMap.get(tableIndex)) * TABLE_BALANCER_WEIGHT;
            float localityIndex = getLocalityIndex(hRegionInfo, server) * LOCALITY_WEIGHT;
            LOG.info("tableRegionaffinity: " + tableRegionAffinityNumber);
            LOG.info("regionAffinityNUmber: " + regionAffinityNumber);
            LOG.info("localityIndex: " + localityIndex);
            double finalAffinity = regionAffinityNumber + tableRegionAffinityNumber + localityIndex
                    + getStickinessWeight(hRegionInfo);
            queue.add(new RegionServerRegionAffinity(server, hRegionInfo, finalAffinity));
            LOG.info("Affinity between server=" + server.getServerName() + " and region="
                    + hRegionInfo.getRegionNameAsString() + " is " + finalAffinity);
        }

        LOG.info("Number of regions to move=" + numberOfRegionsToMove + " All server and region affinities: "
                + queue);

        // Get top numberOfRegionsToMove
        List<RegionServerRegionAffinity> listOfRegionsToMove = new ArrayList<RegionServerRegionAffinity>();
        for (int i = 0; i < numberOfRegionsToMove; ++i) {
            if (queue.isEmpty()) {
                continue;
            }
            listOfRegionsToMove.add(queue.poll());
        }

        // Search for the most affine servers to these listOfRegionsToMove
        for (RegionServerRegionAffinity regionServerRegionAffinity : listOfRegionsToMove) {
            HRegionInfo hRegionInfoToMove = regionServerRegionAffinity.getHRegionInfo();
            ServerAndLoad serverToMove = null;
            double maxAffinity = Double.MIN_VALUE;
            // Get the most affine server to hRegionInfoToMove
            for (ServerAndLoad activeServer : serversByLoad.keySet()) {
                hRegionInfos = serversByLoad.get(activeServer);
                if (activeServer.equals(regionServerRegionAffinity.getServer())) {
                    continue;
                }
                if (hRegionInfos.size() >= ceiling) {
                    LOG.debug("Number of HRegions >= ceiling (" + hRegionInfos.size() + " >= " + ceiling + ")");
                    continue;
                }
                regionAffinityNumber = (1 - hRegionInfos.size() / numRegions) * SERVER_BALANCER_WEIGHT;
                TableName table = hRegionInfoToMove.getTable();
                String tableNameAsString = table.getNameAsString();
                int serverIndex = cluster.serversToIndex.get(activeServer.getServerName().getHostAndPort());
                tableRegionAffinityNumber = 0;
                if (cluster.tablesToIndex.containsKey(tableNameAsString)) {
                    Integer tableIndex = cluster.tablesToIndex.get(tableNameAsString);
                    tableRegionAffinityNumber = (1
                            - cluster.numRegionsPerServerPerTable[serverIndex][tableIndex]
                                    / allTableRegionNumberMap.get(tableIndex))
                            * TABLE_BALANCER_WEIGHT;
                } else {
                    LOG.error("Table " + tableNameAsString + "not present in cluster.tablesToIndex");
                }
                double finalAffinity = regionAffinityNumber + tableRegionAffinityNumber
                        + getLocalityIndex(hRegionInfoToMove, activeServer) * LOCALITY_WEIGHT
                        + getStickinessWeight(hRegionInfoToMove);
                if (finalAffinity > maxAffinity) {
                    maxAffinity = finalAffinity;
                    serverToMove = activeServer;
                }
            }
            regionsToReturn.add(new RegionPlan(hRegionInfoToMove,
                    regionServerRegionAffinity.getServer().getServerName(), serverToMove.getServerName()));
        }
    }

    LOG.info("Returning plan: " + regionsToReturn);

    // Reset previuosly moved regions and add new regions
    previouslyMovedRegions.clear();
    for (RegionPlan regionPlan : regionsToReturn) {
        previouslyMovedRegions.add(regionPlan.getRegionInfo());
    }

    long endTime = System.currentTimeMillis();
    LOG.info("Calculated a load balance in " + (endTime - startTime) + "ms. " + "Moving "
            + regionsToReturn.size() + " regions");
    return regionsToReturn;
}

From source file:org.apache.hadoop.hbase.master.handler.TableEventHandler.java

public boolean reOpenAllRegions(List<HRegionInfo> regions) throws IOException {
    boolean done = false;
    LOG.info("Bucketing regions by region server...");
    HTable table = new HTable(masterServices.getConfiguration(), tableName);
    TreeMap<ServerName, List<HRegionInfo>> serverToRegions = Maps.newTreeMap();
    NavigableMap<HRegionInfo, ServerName> hriHserverMapping;
    try {/* w  ww  . j a  va  2  s  .  c  o m*/
        hriHserverMapping = table.getRegionLocations();
    } finally {
        table.close();
    }

    List<HRegionInfo> reRegions = new ArrayList<HRegionInfo>();
    for (HRegionInfo hri : regions) {
        ServerName rsLocation = hriHserverMapping.get(hri);

        // Skip the offlined split parent region
        // See HBASE-4578 for more information.
        if (null == rsLocation) {
            LOG.info("Skip " + hri);
            continue;
        }
        if (!serverToRegions.containsKey(rsLocation)) {
            LinkedList<HRegionInfo> hriList = Lists.newLinkedList();
            serverToRegions.put(rsLocation, hriList);
        }
        reRegions.add(hri);
        serverToRegions.get(rsLocation).add(hri);
    }

    LOG.info("Reopening " + reRegions.size() + " regions on " + serverToRegions.size() + " region servers.");
    this.masterServices.getAssignmentManager().setRegionsToReopen(reRegions);
    BulkReOpen bulkReopen = new BulkReOpen(this.server, serverToRegions,
            this.masterServices.getAssignmentManager());
    while (true) {
        try {
            if (bulkReopen.bulkReOpen()) {
                done = true;
                break;
            } else {
                LOG.warn("Timeout before reopening all regions");
            }
        } catch (InterruptedException e) {
            LOG.warn("Reopen was interrupted");
            // Preserve the interrupt.
            Thread.currentThread().interrupt();
            break;
        }
    }
    return done;
}

From source file:org.apache.hadoop.hbase.master.procedure.MasterDDLOperationHelper.java

/**
 * Reopen all regions from a table after a schema change operation.
 **//*from   w  w  w . j  av a2s .co m*/
public static boolean reOpenAllRegions(final MasterProcedureEnv env, final TableName tableName,
        final List<HRegionInfo> regionInfoList) throws IOException {
    boolean done = false;
    LOG.info("Bucketing regions by region server...");
    List<HRegionLocation> regionLocations = null;
    Connection connection = env.getMasterServices().getConnection();
    try (RegionLocator locator = connection.getRegionLocator(tableName)) {
        regionLocations = locator.getAllRegionLocations();
    }
    // Convert List<HRegionLocation> to Map<HRegionInfo, ServerName>.
    NavigableMap<HRegionInfo, ServerName> hri2Sn = new TreeMap<HRegionInfo, ServerName>();
    for (HRegionLocation location : regionLocations) {
        hri2Sn.put(location.getRegionInfo(), location.getServerName());
    }
    TreeMap<ServerName, List<HRegionInfo>> serverToRegions = Maps.newTreeMap();
    List<HRegionInfo> reRegions = new ArrayList<HRegionInfo>();
    for (HRegionInfo hri : regionInfoList) {
        ServerName sn = hri2Sn.get(hri);
        // Skip the offlined split parent region
        // See HBASE-4578 for more information.
        if (null == sn) {
            LOG.info("Skip " + hri);
            continue;
        }
        if (!serverToRegions.containsKey(sn)) {
            LinkedList<HRegionInfo> hriList = Lists.newLinkedList();
            serverToRegions.put(sn, hriList);
        }
        reRegions.add(hri);
        serverToRegions.get(sn).add(hri);
    }

    LOG.info("Reopening " + reRegions.size() + " regions on " + serverToRegions.size() + " region servers.");
    AssignmentManager am = env.getMasterServices().getAssignmentManager();
    am.setRegionsToReopen(reRegions);
    BulkReOpen bulkReopen = new BulkReOpen(env.getMasterServices(), serverToRegions, am);
    while (true) {
        try {
            if (bulkReopen.bulkReOpen()) {
                done = true;
                break;
            } else {
                LOG.warn("Timeout before reopening all regions");
            }
        } catch (InterruptedException e) {
            LOG.warn("Reopen was interrupted");
            // Preserve the interrupt.
            Thread.currentThread().interrupt();
            break;
        }
    }
    return done;
}

From source file:org.apache.hadoop.hbase.master.RegionManager.java

static MetaRegion getMetaRegionPointingTo(NavigableMap<byte[], MetaRegion> metaRegions, HRegionInfo newRegion) {
    if (metaRegions.isEmpty()) {
        return null;
    } else if (metaRegions.size() == 1) {
        return metaRegions.get(metaRegions.firstKey());
    } else {// w w  w.  jav  a 2  s.com
        if (metaRegions.containsKey(newRegion.getRegionName())) {
            return metaRegions.get(newRegion.getRegionName());
        }
        return metaRegions.get(metaRegions.headMap(newRegion.getRegionName()).lastKey());
    }
}

From source file:org.apache.hadoop.hbase.MetaTableAccessor.java

/**
 * Returns an HRegionLocationList extracted from the result.
 * @return an HRegionLocationList containing all locations for the region range or null if
 *  we can't deserialize the result.// ww w. ja  va 2 s.c  om
 */
@Nullable
public static RegionLocations getRegionLocations(final Result r) {
    if (r == null)
        return null;
    HRegionInfo regionInfo = getHRegionInfo(r, getRegionInfoColumn());
    if (regionInfo == null)
        return null;

    List<HRegionLocation> locations = new ArrayList<HRegionLocation>(1);
    NavigableMap<byte[], NavigableMap<byte[], byte[]>> familyMap = r.getNoVersionMap();

    locations.add(getRegionLocation(r, regionInfo, 0));

    NavigableMap<byte[], byte[]> infoMap = familyMap.get(getCatalogFamily());
    if (infoMap == null)
        return new RegionLocations(locations);

    // iterate until all serverName columns are seen
    int replicaId = 0;
    byte[] serverColumn = getServerColumn(replicaId);
    SortedMap<byte[], byte[]> serverMap = infoMap.tailMap(serverColumn, false);
    if (serverMap.isEmpty())
        return new RegionLocations(locations);

    for (Map.Entry<byte[], byte[]> entry : serverMap.entrySet()) {
        replicaId = parseReplicaIdFromServerColumn(entry.getKey());
        if (replicaId < 0) {
            break;
        }
        HRegionLocation location = getRegionLocation(r, regionInfo, replicaId);
        // In case the region replica is newly created, it's location might be null. We usually do not
        // have HRL's in RegionLocations object with null ServerName. They are handled as null HRLs.
        if (location == null || location.getServerName() == null) {
            locations.add(null);
        } else {
            locations.add(location);
        }
    }

    return new RegionLocations(locations);
}

From source file:org.apache.hadoop.hbase.replication.ScopeWALEntryFilter.java

@Override
public Entry filter(Entry entry) {
    NavigableMap<byte[], Integer> scopes = entry.getKey().getReplicationScopes();
    if (scopes == null || scopes.isEmpty()) {
        return null;
    }//from   w ww .  ja  v a 2 s  .  co m
    ArrayList<Cell> cells = entry.getEdit().getCells();
    int size = cells.size();
    byte[] fam;
    for (int i = size - 1; i >= 0; i--) {
        Cell cell = cells.get(i);
        // If a bulk load entry has a scope then that means user has enabled replication for bulk load
        // hfiles.
        // TODO There is a similar logic in TableCfWALEntryFilter but data structures are different so
        // cannot refactor into one now, can revisit and see if any way to unify them.
        if (CellUtil.matchingColumn(cell, WALEdit.METAFAMILY, WALEdit.BULK_LOAD)) {
            Cell filteredBulkLoadEntryCell = filterBulkLoadEntries(scopes, cell);
            if (filteredBulkLoadEntryCell != null) {
                cells.set(i, filteredBulkLoadEntryCell);
            } else {
                cells.remove(i);
            }
        } else {
            // The scope will be null or empty if
            // there's nothing to replicate in that WALEdit
            fam = CellUtil.cloneFamily(cell);
            if (!scopes.containsKey(fam) || scopes.get(fam) == HConstants.REPLICATION_SCOPE_LOCAL) {
                cells.remove(i);
            }
        }
    }
    if (cells.size() < size / 2) {
        cells.trimToSize();
    }
    return entry;
}

From source file:org.apache.hadoop.hbase.replication.ScopeWALEntryFilter.java

private Cell filterBulkLoadEntries(NavigableMap<byte[], Integer> scopes, Cell cell) {
    byte[] fam;//  ww  w  . jav a2s.  c o  m
    BulkLoadDescriptor bld = null;
    try {
        bld = WALEdit.getBulkLoadDescriptor(cell);
    } catch (IOException e) {
        LOG.warn("Failed to get bulk load events information from the WAL file.", e);
        return cell;
    }
    List<StoreDescriptor> storesList = bld.getStoresList();
    // Copy the StoreDescriptor list and update it as storesList is a unmodifiableList
    List<StoreDescriptor> copiedStoresList = new ArrayList<StoreDescriptor>(storesList);
    Iterator<StoreDescriptor> copiedStoresListIterator = copiedStoresList.iterator();
    boolean anyStoreRemoved = false;
    while (copiedStoresListIterator.hasNext()) {
        StoreDescriptor sd = copiedStoresListIterator.next();
        fam = sd.getFamilyName().toByteArray();
        if (!scopes.containsKey(fam) || scopes.get(fam) == HConstants.REPLICATION_SCOPE_LOCAL) {
            copiedStoresListIterator.remove();
            anyStoreRemoved = true;
        }
    }

    if (!anyStoreRemoved) {
        return cell;
    } else if (copiedStoresList.isEmpty()) {
        return null;
    }
    BulkLoadDescriptor.Builder newDesc = BulkLoadDescriptor.newBuilder().setTableName(bld.getTableName())
            .setEncodedRegionName(bld.getEncodedRegionName()).setBulkloadSeqNum(bld.getBulkloadSeqNum());
    newDesc.addAllStores(copiedStoresList);
    BulkLoadDescriptor newBulkLoadDescriptor = newDesc.build();
    return CellUtil.createCell(CellUtil.cloneRow(cell), WALEdit.METAFAMILY, WALEdit.BULK_LOAD,
            cell.getTimestamp(), cell.getTypeByte(), newBulkLoadDescriptor.toByteArray());
}