com.linkedin.pinot.broker.routing.HelixExternalViewBasedRouting.java Source code

Java tutorial

Introduction

Here is the source code for com.linkedin.pinot.broker.routing.HelixExternalViewBasedRouting.java

Source

/**
 * Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.linkedin.pinot.broker.routing;

import com.google.common.collect.Sets;
import com.linkedin.pinot.broker.routing.builder.RoutingTableBuilder;
import com.linkedin.pinot.common.config.TableConfig;
import com.linkedin.pinot.common.config.TableNameBuilder;
import com.linkedin.pinot.common.metrics.BrokerMeter;
import com.linkedin.pinot.common.metrics.BrokerMetrics;
import com.linkedin.pinot.common.metrics.BrokerTimer;
import com.linkedin.pinot.common.utils.CommonConstants;
import com.linkedin.pinot.common.utils.EqualityUtils;
import com.linkedin.pinot.common.utils.NetUtil;
import com.linkedin.pinot.common.utils.helix.HelixHelper;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import org.apache.commons.configuration.Configuration;
import org.apache.helix.AccessOption;
import org.apache.helix.HelixDataAccessor;
import org.apache.helix.HelixManager;
import org.apache.helix.PropertyKey;
import org.apache.helix.ZNRecord;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.InstanceConfig;
import org.apache.helix.store.zk.ZkHelixPropertyStore;
import org.apache.zookeeper.data.Stat;
import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/*
 * TODO
 * Would be better to not have the external view based routing not aware of the fact that there are HLC and LLC
 * implementations. A better way to do it would be to have a RoutingTable implementation that merges the output
 * of an offline routing table and a realtime routing table, with the realtime routing table being aware of the
 * fact that there is both an hlc and llc one.
 */
public class HelixExternalViewBasedRouting implements RoutingTable {
    private static final Logger LOGGER = LoggerFactory.getLogger(HelixExternalViewBasedRouting.class);

    private final Map<String, RoutingTableBuilder> _routingTableBuilderMap;

    private final Map<String, Integer> _lastKnownExternalViewVersionMap = new ConcurrentHashMap<>();
    private final Map<String, Map<String, InstanceConfig>> _lastKnownInstanceConfigsForTable = new ConcurrentHashMap<>();
    private final Map<String, InstanceConfig> _lastKnownInstanceConfigs = new ConcurrentHashMap<>();
    private final Map<String, Set<String>> _tablesForInstance = new ConcurrentHashMap<>();

    private final HelixExternalViewBasedTimeBoundaryService _timeBoundaryService;
    private final HelixManager _helixManager;
    private static final int INVALID_EXTERNAL_VIEW_VERSION = Integer.MIN_VALUE;

    private BrokerMetrics _brokerMetrics;

    private Configuration _configuration;

    private ZkHelixPropertyStore<ZNRecord> _propertyStore;

    private RoutingTableBuilderFactory _routingTableBuilderFactory;

    public HelixExternalViewBasedRouting(ZkHelixPropertyStore<ZNRecord> propertyStore, HelixManager helixManager,
            Configuration configuration) {
        _propertyStore = propertyStore;
        _configuration = configuration;
        _timeBoundaryService = new HelixExternalViewBasedTimeBoundaryService(propertyStore);
        _routingTableBuilderMap = new HashMap<>();
        _helixManager = helixManager;
        _routingTableBuilderFactory = new RoutingTableBuilderFactory(_configuration, propertyStore);
    }

    @Override
    public Map<String, List<String>> getRoutingTable(RoutingTableLookupRequest request) {
        String tableName = request.getTableName();
        RoutingTableBuilder routingTableBuilder = _routingTableBuilderMap.get(tableName);
        return routingTableBuilder.getRoutingTable(request);
    }

    @Override
    public boolean routingTableExists(String tableName) {
        return _routingTableBuilderMap.containsKey(tableName);
    }

    public void setBrokerMetrics(BrokerMetrics brokerMetrics) {
        _brokerMetrics = brokerMetrics;
    }

    public void markDataResourceOnline(TableConfig tableConfig, ExternalView externalView,
            List<InstanceConfig> instanceConfigList) {
        String tableName = tableConfig.getTableName();

        RoutingTableBuilder routingTableBuilder = _routingTableBuilderFactory
                .createRoutingTableBuilder(tableConfig);
        routingTableBuilder.init(_configuration, tableConfig, _propertyStore);
        LOGGER.info("Initialized routingTableBuilder: {} for table {}", routingTableBuilder.getClass().getName(),
                tableName);
        _routingTableBuilderMap.put(tableName, routingTableBuilder);

        // Build the routing table
        if (externalView == null) {
            // It is possible for us to get a request to serve a table for which there is no external view. In this case, just
            // keep a bogus last seen external view version to force a rebuild the next time we see an external view.
            _lastKnownExternalViewVersionMap.put(tableName, INVALID_EXTERNAL_VIEW_VERSION);
            return;
        }
        buildRoutingTable(tableName, externalView, instanceConfigList);
    }

    private boolean isRoutingTableRebuildRequired(String tableName, ExternalView externalView,
            List<InstanceConfig> instanceConfigs) {
        // In unit tests, always rebuild the routing table
        if (_helixManager == null) {
            return true;
        }

        // Do we know about this table?
        if (!_lastKnownExternalViewVersionMap.containsKey(tableName)) {
            LOGGER.info("Routing table for table {} requires rebuild due to it being newly added", tableName);
            return true;
        }

        // Check if the znode version changed
        int externalViewRecordVersion = externalView.getRecord().getVersion();
        int lastKnownExternalViewVersion = _lastKnownExternalViewVersionMap.get(tableName);

        if (externalViewRecordVersion != lastKnownExternalViewVersion
                || lastKnownExternalViewVersion == INVALID_EXTERNAL_VIEW_VERSION) {
            LOGGER.info(
                    "Routing table for table {} requires rebuild due to external view change (current version {}, last known version {})",
                    tableName, externalViewRecordVersion, lastKnownExternalViewVersion);
            return true;
        }

        // Check if there are relevant instance config changes
        Map<String, InstanceConfig> lastKnownInstanceConfigs = _lastKnownInstanceConfigsForTable.get(tableName);
        if (lastKnownInstanceConfigs == null || lastKnownInstanceConfigs.isEmpty()) {
            LOGGER.info("Routing table for table {} requires rebuild due to empty/null previous instance configs",
                    tableName);
            return true;
        }

        // Gather relevant incoming instance configs
        Map<String, InstanceConfig> currentRelevantInstanceConfigs = new HashMap<>();
        for (InstanceConfig incomingInstanceConfig : instanceConfigs) {
            String instanceName = incomingInstanceConfig.getInstanceName();

            if (lastKnownInstanceConfigs.containsKey(instanceName)) {
                currentRelevantInstanceConfigs.put(instanceName, incomingInstanceConfig);
            }
        }

        // Did some instances lose their configuration?
        if (lastKnownInstanceConfigs.size() != currentRelevantInstanceConfigs.size()) {
            LOGGER.info(
                    "Routing table for table {} requires rebuild due to having a different number of instance configs (known instance config count {}, current instance config count {})",
                    tableName, lastKnownInstanceConfigs.size(), currentRelevantInstanceConfigs.size());
            return true;
        }

        // Did some instance change state?
        for (String instanceName : lastKnownInstanceConfigs.keySet()) {
            InstanceConfig previousInstanceConfig = lastKnownInstanceConfigs.get(instanceName);
            InstanceConfig currentInstanceConfig = currentRelevantInstanceConfigs.get(instanceName);

            // If it's the same znode, don't bother comparing the contents of the instance configs
            if (previousInstanceConfig.getRecord().getVersion() == currentInstanceConfig.getRecord().getVersion()) {
                continue;
            }

            // Check if the instance got enabled/disabled or started/stopped shutting down since the last update
            boolean wasEnabled = previousInstanceConfig.getInstanceEnabled();
            boolean isEnabled = currentInstanceConfig.getInstanceEnabled();

            String wasShuttingDown = previousInstanceConfig.getRecord()
                    .getSimpleField(CommonConstants.Helix.IS_SHUTDOWN_IN_PROGRESS);
            String isShuttingDown = currentInstanceConfig.getRecord()
                    .getSimpleField(CommonConstants.Helix.IS_SHUTDOWN_IN_PROGRESS);

            boolean instancesChanged = !EqualityUtils.isEqual(wasEnabled, isEnabled)
                    || !EqualityUtils.isEqual(wasShuttingDown, isShuttingDown);

            if (instancesChanged) {
                LOGGER.info(
                        "Routing table for table {} requires rebuild due to at least one instance changing state (instance {} enabled: {} -> {}; shutting down {} -> {})",
                        tableName, instanceName, wasEnabled, isEnabled, wasShuttingDown, isShuttingDown);
                return true;
            } else {
                // Update the instance config in our last known instance config, since it hasn't changed
                _lastKnownInstanceConfigs.put(instanceName, currentInstanceConfig);
                for (String tableForInstance : _tablesForInstance.get(instanceName)) {
                    _lastKnownInstanceConfigsForTable.get(tableForInstance).put(instanceName,
                            currentInstanceConfig);
                }
            }
        }

        // No relevant changes, no need to update the routing table
        LOGGER.info("Routing table for table {} does not require a rebuild", tableName);
        return false;
    }

    private void buildRoutingTable(String tableNameWithType, ExternalView externalView,
            List<InstanceConfig> instanceConfigs) {
        // Save the current version number of the external view to avoid unnecessary routing table updates
        int externalViewRecordVersion = externalView.getRecord().getVersion();
        _lastKnownExternalViewVersionMap.put(tableNameWithType, externalViewRecordVersion);

        RoutingTableBuilder routingTableBuilder = _routingTableBuilderMap.get(tableNameWithType);
        if (routingTableBuilder == null) {
            //TODO: warn
            return;
        }
        CommonConstants.Helix.TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableNameWithType);

        LOGGER.info("Trying to compute routing table for table {} using {}", tableNameWithType,
                routingTableBuilder);
        long startTimeMillis = System.currentTimeMillis();

        try {
            Map<String, InstanceConfig> relevantInstanceConfigs = new HashMap<>();

            routingTableBuilder.computeRoutingTableFromExternalView(tableNameWithType, externalView,
                    instanceConfigs);

            // Keep track of the instance configs that are used in that routing table
            updateInstanceConfigsMapFromExternalView(relevantInstanceConfigs, instanceConfigs, externalView);

            // Save the instance configs used so that we can avoid unnecessary routing table updates later
            _lastKnownInstanceConfigsForTable.put(tableNameWithType, relevantInstanceConfigs);
            for (InstanceConfig instanceConfig : relevantInstanceConfigs.values()) {
                _lastKnownInstanceConfigs.put(instanceConfig.getInstanceName(), instanceConfig);
            }

            // Ensure this table is registered with all relevant instances
            for (String instanceName : relevantInstanceConfigs.keySet()) {
                Set<String> tablesForCurrentInstance = _tablesForInstance.get(instanceName);

                // Ensure there is a table set for this instance
                if (tablesForCurrentInstance == null) {
                    synchronized (_tablesForInstance) {
                        if (!_tablesForInstance.containsKey(instanceName)) {
                            tablesForCurrentInstance = Sets.newConcurrentHashSet();
                            _tablesForInstance.put(instanceName, tablesForCurrentInstance);
                        } else {
                            // Another thread has created a table set for this instance, use it
                            tablesForCurrentInstance = _tablesForInstance.get(instanceName);
                        }
                    }
                }

                // Add the table to the set of tables for this instance
                tablesForCurrentInstance.add(tableNameWithType);
            }
        } catch (Exception e) {
            _brokerMetrics.addMeteredTableValue(tableNameWithType, BrokerMeter.ROUTING_TABLE_REBUILD_FAILURES, 1L);
            LOGGER.error("Failed to compute/update the routing table", e);

            // Mark the routing table as needing a rebuild
            _lastKnownExternalViewVersionMap.put(tableNameWithType, INVALID_EXTERNAL_VIEW_VERSION);
        }

        try {
            // We need to compute the time boundary only in two situations:
            // 1) We're adding/updating an offline table and there's a realtime table that we're serving
            // 2) We're adding a new realtime table and there's already an offline table, in which case we need to update the
            //    time boundary for the existing offline table
            String tableForTimeBoundaryUpdate = null;
            ExternalView externalViewForTimeBoundaryUpdate = null;

            if (tableType == CommonConstants.Helix.TableType.OFFLINE) {
                // Does a realtime table exist?
                String realtimeTableName = TableNameBuilder.REALTIME
                        .tableNameWithType(TableNameBuilder.extractRawTableName(tableNameWithType));
                if (_routingTableBuilderMap.containsKey(realtimeTableName)) {
                    tableForTimeBoundaryUpdate = tableNameWithType;
                    externalViewForTimeBoundaryUpdate = externalView;
                }
            }

            if (tableType == CommonConstants.Helix.TableType.REALTIME) {
                // Does an offline table exist?
                String offlineTableName = TableNameBuilder.OFFLINE
                        .tableNameWithType(TableNameBuilder.extractRawTableName(tableNameWithType));
                if (_routingTableBuilderMap.containsKey(offlineTableName)) {
                    // Is there no time boundary?
                    if (_timeBoundaryService.getTimeBoundaryInfoFor(offlineTableName) == null) {
                        tableForTimeBoundaryUpdate = offlineTableName;
                        externalViewForTimeBoundaryUpdate = fetchExternalView(offlineTableName);
                    }
                }
            }

            if (tableForTimeBoundaryUpdate != null) {
                updateTimeBoundary(tableForTimeBoundaryUpdate, externalViewForTimeBoundaryUpdate);
            } else {
                LOGGER.info("No need to update time boundary for table {}", tableNameWithType);
            }
        } catch (Exception e) {
            LOGGER.error("Failed to update the TimeBoundaryService", e);
        }

        long updateTime = System.currentTimeMillis() - startTimeMillis;

        if (_brokerMetrics != null) {
            _brokerMetrics.addTimedValue(BrokerTimer.ROUTING_TABLE_UPDATE_TIME, updateTime, TimeUnit.MILLISECONDS);
        }

        LOGGER.info("Routing table update for table {} completed in {} ms", tableNameWithType, updateTime);
    }

    protected void updateTimeBoundary(String tableName, ExternalView externalView) {
        LOGGER.info("Trying to compute time boundary service for table {}", tableName);
        long timeBoundaryUpdateStart = System.currentTimeMillis();
        _timeBoundaryService.updateTimeBoundaryService(externalView);
        long timeBoundaryUpdateEnd = System.currentTimeMillis();
        LOGGER.info("Computed the time boundary for table {} in {} ms", tableName,
                (timeBoundaryUpdateEnd - timeBoundaryUpdateStart));
    }

    protected ExternalView fetchExternalView(String table) {
        return HelixHelper.getExternalViewForResource(_helixManager.getClusterManagmentTool(),
                _helixManager.getClusterName(), table);
    }

    private void updateInstanceConfigsMapFromExternalView(Map<String, InstanceConfig> relevantInstanceConfigs,
            List<InstanceConfig> instanceConfigs, ExternalView externalView) {
        Set<String> relevantInstanceNames = new HashSet<>();

        // Gather all the instance names contained in the external view
        for (String partitionName : externalView.getPartitionSet()) {
            relevantInstanceNames.addAll(externalView.getStateMap(partitionName).keySet());
        }

        // Update the relevant instance config map with the instance configs given
        for (InstanceConfig instanceConfig : instanceConfigs) {
            if (relevantInstanceNames.contains(instanceConfig.getInstanceName())) {
                relevantInstanceConfigs.put(instanceConfig.getInstanceName(), instanceConfig);
            }
        }
    }

    public void markDataResourceOffline(String tableName) {
        LOGGER.info("Trying to remove data table from broker for {}", tableName);
        _routingTableBuilderMap.remove(tableName);
        _lastKnownExternalViewVersionMap.remove(tableName);
        _lastKnownInstanceConfigsForTable.remove(tableName);
        _timeBoundaryService.remove(tableName);

        // Remove table from all instances
        synchronized (_tablesForInstance) {
            Iterator<Map.Entry<String, Set<String>>> iterator = _tablesForInstance.entrySet().iterator();
            while (iterator.hasNext()) {
                Map.Entry<String, Set<String>> entry = iterator.next();
                entry.getValue().remove(tableName);
                if (entry.getValue().isEmpty()) {
                    _lastKnownInstanceConfigs.remove(entry.getKey());
                    iterator.remove();
                }
            }

        }
    }

    public void processExternalViewChange() {
        long startTime = System.currentTimeMillis();

        // Get list of tables that we're serving
        List<String> tablesServed = new ArrayList<>(_lastKnownExternalViewVersionMap.keySet());

        if (tablesServed.isEmpty()) {
            return;
        }

        // Build list of external views to fetch
        HelixDataAccessor helixDataAccessor = _helixManager.getHelixDataAccessor();
        PropertyKey.Builder propertyKeyBuilder = helixDataAccessor.keyBuilder();

        List<String> externalViewPaths = new ArrayList<>(tablesServed.size());
        for (String tableName : tablesServed) {
            PropertyKey propertyKey = propertyKeyBuilder.externalView(tableName);
            externalViewPaths.add(propertyKey.getPath());
        }

        // Get znode stats for all tables that we're serving
        long statStartTime = System.currentTimeMillis();
        Stat[] externalViewStats = helixDataAccessor.getBaseDataAccessor().getStats(externalViewPaths,
                AccessOption.PERSISTENT);
        long statEndTime = System.currentTimeMillis();

        // Make a list of external views that changed
        List<String> tablesThatChanged = new ArrayList<>();

        long evCheckStartTime = System.currentTimeMillis();
        for (int i = 0; i < externalViewStats.length; i++) {
            Stat externalViewStat = externalViewStats[i];
            if (externalViewStat != null) {
                String currentTableName = tablesServed.get(i);
                int currentExternalViewVersion = externalViewStat.getVersion();
                int lastKnownExternalViewVersion = _lastKnownExternalViewVersionMap.get(currentTableName);

                if (lastKnownExternalViewVersion != currentExternalViewVersion) {
                    tablesThatChanged.add(currentTableName);
                }
            }
        }
        long evCheckEndTime = System.currentTimeMillis();

        // Fetch the instance configs and update the routing tables for the tables that changed
        long icFetchTime = 0;
        long rebuildStartTime = System.currentTimeMillis();
        if (!tablesThatChanged.isEmpty()) {
            // Fetch instance configs
            long icFetchStart = System.currentTimeMillis();
            List<InstanceConfig> instanceConfigs = helixDataAccessor
                    .getChildValues(propertyKeyBuilder.instanceConfigs());
            long icFetchEnd = System.currentTimeMillis();
            icFetchTime = icFetchEnd - icFetchStart;

            for (String tableThatChanged : tablesThatChanged) {
                // We ignore the external views given by Helix on external view change and fetch the latest version as our
                // version of Helix (0.6.5) does not batch external view change messages.
                ExternalView externalView = helixDataAccessor
                        .getProperty(propertyKeyBuilder.externalView(tableThatChanged));

                buildRoutingTable(tableThatChanged, externalView, instanceConfigs);
            }
        }
        long rebuildEndTime = System.currentTimeMillis();

        long endTime = System.currentTimeMillis();
        LOGGER.info(
                "Processed external view change in {} ms (stat {} ms, EV check {} ms, IC fetch {} ms, rebuild {} ms), routing tables rebuilt for tables {}, {} / {} routing tables rebuilt",
                (endTime - startTime), (statEndTime - statStartTime), (evCheckEndTime - evCheckStartTime),
                icFetchTime, (rebuildEndTime - rebuildStartTime), tablesThatChanged, tablesThatChanged.size(),
                tablesServed.size());
    }

    public void processInstanceConfigChange() {
        long startTime = System.currentTimeMillis();

        // Get stats for all relevant instance configs
        HelixDataAccessor helixDataAccessor = _helixManager.getHelixDataAccessor();
        PropertyKey.Builder propertyKeyBuilder = helixDataAccessor.keyBuilder();
        List<String> instancesUsed = new ArrayList<>(_tablesForInstance.keySet());
        List<String> instancePaths = new ArrayList<>(instancesUsed.size());

        for (String instanceName : instancesUsed) {
            PropertyKey propertyKey = propertyKeyBuilder.instanceConfig(instanceName);
            instancePaths.add(propertyKey.getPath());
        }

        if (instancePaths.isEmpty()) {
            return;
        }

        long statFetchStart = System.currentTimeMillis();
        Map<String, Stat> instanceConfigStatMap = new HashMap<>();
        Stat[] instanceConfigStats = helixDataAccessor.getBaseDataAccessor().getStats(instancePaths,
                AccessOption.PERSISTENT);
        long statFetchEnd = System.currentTimeMillis();

        // Make a list of instance configs that changed
        long icConfigCheckStart = System.currentTimeMillis();
        List<String> instancesThatChanged = new ArrayList<>();

        for (int i = 0; i < instanceConfigStats.length; i++) {
            Stat instanceConfigStat = instanceConfigStats[i];
            if (instanceConfigStat != null) {
                String instanceName = instancesUsed.get(i);
                int currentInstanceConfigVersion = instanceConfigStat.getVersion();
                int lastKnownInstanceConfigVersion = _lastKnownInstanceConfigs.get(instanceName).getRecord()
                        .getVersion();

                if (currentInstanceConfigVersion != lastKnownInstanceConfigVersion) {
                    instancesThatChanged.add(instanceName);
                }
                instanceConfigStatMap.put(instanceName, instanceConfigStat);
            }
        }

        // Make a list of all tables affected by the instance config changes
        Set<String> affectedTables = new HashSet<>();
        for (String instanceName : instancesThatChanged) {
            affectedTables.addAll(_tablesForInstance.get(instanceName));
        }
        long icConfigCheckEnd = System.currentTimeMillis();

        // Update the routing tables
        long icFetchTime = 0;
        long evFetchTime = 0;
        long rebuildCheckTime = 0;
        long buildTime = 0;
        int routingTablesRebuiltCount = 0;
        if (!affectedTables.isEmpty()) {
            long icFetchStart = System.currentTimeMillis();
            List<InstanceConfig> instanceConfigs = helixDataAccessor
                    .getChildValues(propertyKeyBuilder.instanceConfigs());
            // Helix does not set the version field, we need to set it explicitly
            for (InstanceConfig instanceConfig : instanceConfigs) {
                Stat stat = instanceConfigStatMap.get(instanceConfig.getInstanceName());
                if (stat != null) {
                    instanceConfig.getRecord().setVersion(stat.getVersion());
                }
            }
            long icFetchEnd = System.currentTimeMillis();
            icFetchTime = icFetchEnd - icFetchStart;

            for (String tableName : affectedTables) {
                long evFetchStart = System.currentTimeMillis();
                ExternalView externalView = helixDataAccessor
                        .getProperty(propertyKeyBuilder.externalView(tableName));
                long evFetchEnd = System.currentTimeMillis();
                evFetchTime += evFetchEnd - evFetchStart;

                long rebuildCheckStart = System.currentTimeMillis();
                final boolean routingTableRebuildRequired = isRoutingTableRebuildRequired(tableName, externalView,
                        instanceConfigs);
                long rebuildCheckEnd = System.currentTimeMillis();
                rebuildCheckTime += rebuildCheckEnd - rebuildCheckStart;

                if (routingTableRebuildRequired) {
                    long rebuildStart = System.currentTimeMillis();
                    buildRoutingTable(tableName, externalView, instanceConfigs);
                    long rebuildEnd = System.currentTimeMillis();
                    buildTime += rebuildEnd - rebuildStart;
                    routingTablesRebuiltCount++;
                }
            }
        }
        long endTime = System.currentTimeMillis();

        LOGGER.info(
                "Processed instance config change in {} ms (stat {} ms, IC check {} ms, IC fetch {} ms, EV fetch {} ms, rebuild check {} ms, rebuild {} ms), {} / {} routing tables rebuilt",
                (endTime - startTime), (statFetchEnd - statFetchStart), (icConfigCheckEnd - icConfigCheckStart),
                icFetchTime, evFetchTime, rebuildCheckTime, buildTime, routingTablesRebuiltCount,
                _lastKnownExternalViewVersionMap.size());
    }

    public TimeBoundaryService getTimeBoundaryService() {
        return _timeBoundaryService;
    }

    @Override
    public String dumpSnapshot(String tableName) throws Exception {
        JSONObject ret = new JSONObject();
        JSONArray routingTableSnapshot = new JSONArray();

        for (String currentTable : _routingTableBuilderMap.keySet()) {
            if (tableName == null || currentTable.startsWith(tableName)) {
                JSONObject tableEntry = new JSONObject();
                tableEntry.put("tableName", currentTable);

                JSONArray entries = new JSONArray();
                RoutingTableBuilder routingTableBuilder = _routingTableBuilderMap.get(currentTable);
                List<Map<String, List<String>>> routingTables = routingTableBuilder.getRoutingTables();
                for (Map<String, List<String>> routingTable : routingTables) {
                    entries.put(new JSONObject(routingTable.toString()));
                }
                tableEntry.put("routingTableEntries", entries);
                routingTableSnapshot.put(tableEntry);
            }
        }
        ret.put("routingTableSnapshot", routingTableSnapshot);
        ret.put("host", NetUtil.getHostnameOrAddress());

        return ret.toString(2);
    }
}