com.googlecode.icegem.cacheutils.regioncomparator.CompareTool.java Source code

Java tutorial

Introduction

Here is the source code for com.googlecode.icegem.cacheutils.regioncomparator.CompareTool.java

Source

/*
 * Icegem, Extensions library for VMWare vFabric GemFire
 * 
 * Copyright (c) 2010-2011, Grid Dynamics Consulting Services Inc. or third-party  
 * contributors as indicated by the @author tags or express copyright attribution
 * statements applied by the authors.  
 * 
 * This copyrighted material is made available to anyone wishing to use, modify,
 * copy, or redistribute it subject to the terms and conditions of the GNU
 * Lesser General Public License v3, as published by the Free Software Foundation.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 * 
 * You should have received a copy of the GNU Lesser General Public License v3
 * along with this distribution; if not, write to:
 * Free Software Foundation, Inc.
 * 51 Franklin Street, Fifth Floor
 * Boston, MA  02110-1301  USA
 */
package com.googlecode.icegem.cacheutils.regioncomparator;

import java.util.*;
import java.util.concurrent.*;

import com.gemstone.gemfire.admin.AdminDistributedSystem;
import com.gemstone.gemfire.admin.AdminDistributedSystemFactory;
import com.gemstone.gemfire.cache.client.Pool;
import com.gemstone.gemfire.cache.client.PoolManager;
import com.gemstone.gemfire.cache.client.internal.AutoConnectionSourceImpl;
import com.gemstone.gemfire.cache.client.internal.PoolImpl;
import com.gemstone.gemfire.cache.execute.FunctionService;
import com.gemstone.gemfire.cache.execute.ResultCollector;
import com.gemstone.gemfire.distributed.internal.ServerLocation;
import org.apache.commons.cli.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.googlecode.icegem.cacheutils.Tool;
import com.googlecode.icegem.cacheutils.common.Utils;

public class CompareTool extends Tool {
    private static final Logger log = LoggerFactory.getLogger(CompareTool.class);
    private static String serversOption = "";
    private static String regionName;
    private static List<String> scanPackagesOption;
    private static Properties locatorsProperties = new Properties();

    protected void parseCommandLineArguments(String[] commandLineArguments) {
        Options options = constructGnuOptions();
        if (commandLineArguments.length < 1) {
            printHelp(options);
            Utils.exitWithSuccess();
        }
        CommandLineParser parser = new GnuParser();
        try {
            CommandLine line = parser.parse(options, commandLineArguments);
            if (!line.hasOption("region") || line.hasOption("help")
                    || !(line.hasOption("locators") || line.hasOption("servers"))) {
                printHelp(options);
                Utils.exitWithSuccess();
            }
            if (line.hasOption("packages"))
                scanPackagesOption = Arrays.asList(line.getOptionValue("packages").split(","));
            regionName = line.getOptionValue("region");
            if (line.hasOption("locators"))
                locatorsProperties = line.getOptionProperties("locators");
            if (line.hasOption("servers"))
                serversOption = line.getOptionValue("servers");

        } catch (ParseException exp) {
            System.err.println("Parsing options failed. " + exp.getMessage());
            printHelp(options);
            Utils.exitWithSuccess();
        }
    }

    protected void printHelp(final Options options) {
        HelpFormatter formatter = new HelpFormatter();
        formatter.printHelp("compare [options]", options);
    }

    protected Options constructGnuOptions() {
        final Options gnuOptions = new Options();
        Option locators = OptionBuilder.hasArgs().withDescription(
                "Locators of GemFire system. For intra-cluster checking. Example: host1[port1],host2[port2]")
                .withValueSeparator().withArgName("cluster=locators").withLongOpt("locators").create("l");
        gnuOptions.addOption("r", "region", true,
                "Region path to be compared. Only replicated region could be used. Example: /region1/region2")
                .addOption("s", "servers", true,
                        "Servers of GemFire system. For multi-cluster systems. Example: host1[port1],host2[port2]")
                .addOption(locators)
                .addOption("c", "packages", true,
                        "Enumerate packages to scan for @AutoSerializable model classes. Delimiter is a comma sign.")
                .addOption("h", "help", false, "Print usage information");
        return gnuOptions;
    }

    public void execute(String[] args, boolean debugEnabled, boolean quiet) {
        AdminDistributedSystem adminDs = AdminDistributedSystemFactory
                .getDistributedSystem(AdminDistributedSystemFactory.defineDistributedSystem());
        adminDs.connect();

        parseCommandLineArguments(args);

        List<Pool> poolList = new ArrayList<Pool>();
        if (serversOption != null && serversOption.length() > 0)
            for (String serverOption : serversOption.split(",")) {
                String serverHost = serverOption.substring(0, serverOption.indexOf("["));
                String serverPort = serverOption.substring(serverOption.indexOf("[") + 1,
                        serverOption.indexOf("]"));
                poolList.add(PoolManager.createFactory().addServer(serverHost, Integer.parseInt(serverPort))
                        .create("poolTo" + serverHost + serverPort));
            }
        if (locatorsProperties != null && !locatorsProperties.isEmpty())
            for (Object poolOption : locatorsProperties.keySet()) {
                String locator = (String) locatorsProperties.get(poolOption);
                String serverHost = locator.substring(0, locator.indexOf("["));
                String serverPort = locator.substring(locator.indexOf("[") + 1, locator.indexOf("]"));
                poolList.add(PoolManager.createFactory().addLocator(serverHost, Integer.parseInt(serverPort)) //todo: check when we have two identical locators options: exception a pool name already exist
                        .create("poolTo" + serverHost + serverPort));
            }

        //todo: insert checking that each cluster contains region and one's type is equal (Partitioned, Replicated)

        boolean partitioned = false; //todo: insert CLI usage  + throw exception if real region has another type

        List<ServerLocation> serverFromPool = new ArrayList<ServerLocation>();
        List<Pool> emptyPools = new ArrayList<Pool>(); //contains pool with no available servers
        for (Pool pool : poolList) {
            List<ServerLocation> allServers = null;
            if (!pool.getLocators().isEmpty())
                allServers = ((AutoConnectionSourceImpl) ((PoolImpl) pool).getConnectionSource()).findAllServers(); //todo: ConnectionError if locator doesn't exist
            else if (!pool.getServers().isEmpty())
                allServers = Arrays
                        .asList((((PoolImpl) pool).getConnectionSource()).findServer(Collections.emptySet()));

            if (allServers != null)
                serverFromPool.addAll(allServers);
            else {
                log.info("not found servers on locator {}", pool);
                emptyPools.add(pool);
            }
        }
        poolList.removeAll(emptyPools);

        if (serverFromPool.size() == 0) {
            log.info("no servers available");
            return;
        }

        printServerLocationDetails(serverFromPool);

        //source for comparison //todo: if this node doesn't contain region! it's problem
        Pool sourcePool;
        if (!partitioned) {
            int randomServerLocation = new Random().nextInt(serverFromPool.size());
            sourcePool = PoolManager.createFactory()
                    .addServer(serverFromPool.get(randomServerLocation).getHostName(),
                            serverFromPool.get(randomServerLocation).getPort())
                    .create("target");
        } else {
            sourcePool = poolList.get(0);
            poolList.remove(0);
        }

        FunctionService.registerFunction(new RegionInfoFunction());
        ResultCollector regionInfoResult = FunctionService.onServers(sourcePool).withArgs(regionName)
                .execute(new RegionInfoFunction());

        Map regionInfo = (HashMap) ((ArrayList) regionInfoResult.getResult()).get(0);
        System.out.println("region info: " + regionInfo);

        int totalNumBuckets = (Integer) regionInfo.get("totalNumBuckets");
        //log.debug("total keys' batch counts is ", totalNumBuckets);
        System.out.println("total keys' batch counts is " + totalNumBuckets);
        KeyExtractor keyExtractor = new KeyExtractor(regionName, sourcePool, partitioned, totalNumBuckets);

        Map<String, Map<String, Set>> clusterDifference = new HashMap<String, Map<String, Set>>(); //key: memeberId list: absent keys, diff values

        List<PoolResult> taskResults = new ArrayList<PoolResult>();
        List<Future<PoolResult>> collectTasks = new ArrayList<Future<PoolResult>>(poolList.size());
        ExecutorService executorService = Executors.newFixedThreadPool(poolList.size());
        while (keyExtractor.hasKeys()) {
            Set keys = keyExtractor.getNextKeysBatch();
            System.out.println("keys to check: " + keys);
            for (Pool nextPool : poolList)
                collectTasks.add(executorService.submit(new CollectorTask(keys, nextPool, regionName)));
            System.out.println("active tasks: " + collectTasks.size());
            try {
                //for (Future<ResultCollector> futureTask : collectTasks) {
                for (Future<PoolResult> futureTask : collectTasks) {
                    taskResults.add(futureTask.get());
                }
            } catch (InterruptedException ie) {
                ie.printStackTrace();
            } catch (ExecutionException ee) {
                ee.printStackTrace();
            }
            collectTasks.clear();

            System.out.println("compare contents..");
            //getting source contents
            Map sourceData = new HashMap();

            //getting source map
            FutureTask<PoolResult> ft = new FutureTask<PoolResult>(new CollectorTask(keys, sourcePool, regionName));
            ft.run();
            try {
                PoolResult rc = ft.get();
                List poolResult = (List) rc.getResultCollector().getResult();
                for (Object singleResult : poolResult) {
                    sourceData.putAll((Map) ((HashMap) singleResult).get("map"));
                }
            } catch (Exception e) {
                throw new RuntimeException("error getting key-hash from pool: " + sourcePool, e);
            }
            //todo: aggregate members' data from one cluster

            System.out.println("source data is: " + sourceData);
            //for (ResultCollector taskResultFromPool : taskResults) {
            for (PoolResult taskResultFromPool : taskResults) {
                List poolResult = (ArrayList) taskResultFromPool.getResultCollector().getResult();
                if (!partitioned) {
                    for (Object resultFromMember : poolResult) {
                        Map result = (HashMap) resultFromMember;
                        String memberId = (String) result.get("memberId");
                        if (regionInfo.get("id").equals(result.get("memberId"))) //for replicated region
                            continue;
                        Map<String, Set> aggregationInfo = compareAndAggregate(sourceData,
                                (HashMap) result.get("map"));
                        System.out.println("result of comparing is: " + aggregationInfo);
                        if (!clusterDifference.containsKey(memberId)) {
                            aggregationInfo.put("absentKeys", new HashSet());
                            clusterDifference.put(memberId, aggregationInfo);
                        } else {
                            Map<String, Set> difference = clusterDifference.get(memberId);
                            difference.get("absentKeys").addAll((Set) result.get("absentKeys"));
                            difference.get("diffValues").addAll(aggregationInfo.get("diffValues"));
                            clusterDifference.put(memberId, difference);
                        }
                    }
                } else {
                    Map targetData = new HashMap();
                    Set absentKeysFromPool = new HashSet();

                    //aggregate data from different members with partition region
                    for (Object resultFromMember : poolResult) {
                        targetData.putAll((Map) ((HashMap) resultFromMember).get("map"));
                        absentKeysFromPool.addAll((Set) ((HashMap) resultFromMember).get("absentKeys"));
                    }

                    Map<String, Set> aggregationInfo = compareAndAggregate(sourceData, targetData);
                    System.out.println("result of comparing is: " + aggregationInfo);
                    String keyForPartitionRegionType = taskResultFromPool.getPool().toString();
                    if (!clusterDifference.containsKey(keyForPartitionRegionType)) {
                        clusterDifference.put(keyForPartitionRegionType, aggregationInfo);
                    } else {
                        Map<String, Set> difference = clusterDifference.get(keyForPartitionRegionType);
                        difference.get("absentKeys").addAll(aggregationInfo.get("absentKeys"));
                        difference.get("diffValues").addAll(aggregationInfo.get("diffValues"));
                        clusterDifference.put(keyForPartitionRegionType, difference);
                    }
                }
            }

            taskResults.clear();
        }

        System.out.println("____________________________");
        System.out.println("difference: ");
        System.out.println(clusterDifference);
        executorService.shutdown();
        adminDs.disconnect();
    }

    /**
     * compare two snapshots from source and target maps
     * format: key: absentKeys or map
     */
    private static Map<String, Set> compareAndAggregate(Map sourceMap, Map targetMap) {
        //keys witch values are different
        System.out.println("compare maps");
        System.out.println("source: " + sourceMap);
        System.out.println("target: " + targetMap);
        Map<String, Set> aggregationInfo = new HashMap<String, Set>();
        Set<Object> keysForDiffValues = new HashSet<Object>();
        for (Object regionKey : targetMap.keySet()) {
            if (!targetMap.get(regionKey).equals(sourceMap.get(regionKey))) //we compare not original values, but it's hash code
                keysForDiffValues.add(regionKey);
        }

        aggregationInfo.put("diffValues", keysForDiffValues);
        return aggregationInfo;
    }

    private void printServerLocationDetails(List<ServerLocation> serverLocationList) {
        for (ServerLocation server : serverLocationList) {
            System.out.println("host: " + server.getHostName());
            System.out.println("port: " + server.getPort());
            System.out.println("----------------------");
        }
    }
}