org.epics.archiverappliance.config.DefaultConfigService.java Source code

Java tutorial

Introduction

Here is the source code for org.epics.archiverappliance.config.DefaultConfigService.java

Source

/*******************************************************************************
 * Copyright (c) 2011 The Board of Trustees of the Leland Stanford Junior University
 * as Operator of the SLAC National Accelerator Laboratory.
 * Copyright (c) 2011 Brookhaven National Laboratory.
 * EPICS archiver appliance is distributed subject to a Software License Agreement found
 * in file LICENSE that is included with this distribution.
 *******************************************************************************/
package org.epics.archiverappliance.config;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.lang.management.ManagementFactory;
import java.lang.management.PlatformLoggingMXBean;
import java.net.InetAddress;
import java.net.URL;
import java.net.UnknownHostException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListSet;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;

import javax.servlet.ServletContext;

import org.apache.commons.validator.routines.InetAddressValidator;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.epics.archiverappliance.StoragePlugin;
import org.epics.archiverappliance.common.ProcessMetrics;
import org.epics.archiverappliance.common.TimeUtils;
import org.epics.archiverappliance.config.PVTypeInfoEvent.ChangeType;
import org.epics.archiverappliance.config.exception.AlreadyRegisteredException;
import org.epics.archiverappliance.config.exception.ConfigException;
import org.epics.archiverappliance.config.persistence.MySQLPersistence;
import org.epics.archiverappliance.config.pubsub.PubSubEvent;
import org.epics.archiverappliance.engine.ArchiveEngine;
import org.epics.archiverappliance.engine.pv.EngineContext;
import org.epics.archiverappliance.etl.common.PBThreeTierETLPVLookup;
import org.epics.archiverappliance.mgmt.MgmtPostStartup;
import org.epics.archiverappliance.mgmt.MgmtRuntimeState;
import org.epics.archiverappliance.mgmt.NonMgmtPostStartup;
import org.epics.archiverappliance.mgmt.bpl.cahdlers.NamesHandler;
import org.epics.archiverappliance.mgmt.policy.ExecutePolicy;
import org.epics.archiverappliance.mgmt.policy.PolicyConfig;
import org.epics.archiverappliance.mgmt.policy.PolicyConfig.SamplingMethod;
import org.epics.archiverappliance.retrieval.RetrievalState;
import org.epics.archiverappliance.retrieval.channelarchiver.XMLRPCClient;
import org.epics.archiverappliance.utils.ui.GetUrlContent;
import org.epics.archiverappliance.utils.ui.JSONDecoder;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.xml.sax.SAXException;

import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.cache.RemovalListener;
import com.google.common.cache.RemovalNotification;
import com.google.common.eventbus.AsyncEventBus;
import com.google.common.eventbus.EventBus;
import com.google.common.eventbus.Subscribe;
import com.hazelcast.client.HazelcastClient;
import com.hazelcast.client.config.ClientConfig;
import com.hazelcast.config.Config;
import com.hazelcast.config.XmlConfigBuilder;
import com.hazelcast.core.Cluster;
import com.hazelcast.core.EntryEvent;
import com.hazelcast.core.Hazelcast;
import com.hazelcast.core.HazelcastInstance;
import com.hazelcast.core.IMap;
import com.hazelcast.core.ITopic;
import com.hazelcast.core.Member;
import com.hazelcast.core.MemberAttributeEvent;
import com.hazelcast.core.MembershipEvent;
import com.hazelcast.core.MembershipListener;
import com.hazelcast.core.Message;
import com.hazelcast.core.MessageListener;
import com.hazelcast.map.listener.EntryAddedListener;
import com.hazelcast.map.listener.EntryRemovedListener;
import com.hazelcast.map.listener.EntryUpdatedListener;

import edu.stanford.slac.archiverappliance.PB.data.PBTypeSystem;

/**
 * This is the default config service for the archiver appliance.
 * There is a subclass that is used in the junit tests.
 * 
 * @author mshankar
 *
 */
public class DefaultConfigService implements ConfigService {
    private static Logger logger = Logger.getLogger(DefaultConfigService.class.getName());
    private static Logger configlogger = Logger.getLogger("config." + DefaultConfigService.class.getName());
    private static Logger clusterLogger = Logger.getLogger("cluster." + DefaultConfigService.class.getName());

    public static final String SITE_FOR_UNIT_TESTS_NAME = "org.epics.archiverappliance.config.site";
    public static final String SITE_FOR_UNIT_TESTS_VALUE = "tests";

    /**
     * Add a property in archappl.properties under this key to identify the class that implements the PVNameToKeyMapping for this installation.
     */
    public static final String ARCHAPPL_PVNAME_TO_KEY_MAPPING_CLASSNAME = "org.epics.archiverappliance.config.DefaultConfigService.PVName2KeyMappingClassName";

    // Configuration state begins here.
    protected String myIdentity;
    protected ApplianceInfo myApplianceInfo = null;
    protected Map<String, ApplianceInfo> appliances = null;
    // Persisted state begins here.
    protected Map<String, PVTypeInfo> typeInfos = null;
    protected Map<String, UserSpecifiedSamplingParams> archivePVRequests = null;
    protected Map<String, String> channelArchiverDataServers = null;
    protected Map<String, String> aliasNamesToRealNames = null;
    // These are not persisted but derived from other info
    protected Map<String, ApplianceInfo> pv2appliancemapping = null;
    protected Map<String, String> clusterInet2ApplianceIdentity = null;
    protected Map<String, List<ChannelArchiverDataServerPVInfo>> pv2ChannelArchiverDataServer = null;
    protected ITopic<PubSubEvent> pubSub = null;
    protected Map<String, Boolean> namedFlags = null;
    // Configuration state ends here.

    // Runtime state begins here 
    protected LinkedList<Runnable> shutdownHooks = new LinkedList<Runnable>();
    protected PBThreeTierETLPVLookup etlPVLookup = null;
    protected RetrievalState retrievalState = null;
    protected MgmtRuntimeState mgmtRuntime = null;;
    protected EngineContext engineContext = null;
    protected ConcurrentSkipListSet<String> appliancesInCluster = new ConcurrentSkipListSet<String>();
    // Runtime state ends here 

    // This is an optimization; we cache a copy of PVs that are registered for this appliance.
    protected ConcurrentSkipListSet<String> pvsForThisAppliance = null;
    // Maintain a TRIE index for the pvNames in this appliance.
    protected ConcurrentHashMap<String, ConcurrentSkipListSet<String>> parts2PVNamesForThisAppliance = new ConcurrentHashMap<String, ConcurrentSkipListSet<String>>();
    protected ConcurrentSkipListSet<String> pausedPVsForThisAppliance = null;
    protected ApplianceAggregateInfo applianceAggregateInfo = new ApplianceAggregateInfo();
    protected EventBus eventBus = new AsyncEventBus(Executors.newSingleThreadExecutor(new ThreadFactory() {
        @Override
        public Thread newThread(Runnable r) {
            return new Thread(r, "Event bus");
        }
    }));
    protected Properties archapplproperties = new Properties();
    protected PVNameToKeyMapping pvName2KeyConverter = null;
    protected ConfigPersistence persistanceLayer;

    // State local to DefaultConfigService.
    protected WAR_FILE warFile = WAR_FILE.MGMT;
    protected STARTUP_SEQUENCE startupState = STARTUP_SEQUENCE.ZEROTH_STATE;
    protected ScheduledExecutorService startupExecutor = null;
    protected ProcessMetrics processMetrics = new ProcessMetrics();
    private HashSet<String> runTimeFields = new HashSet<String>();
    // Use a Guava cache to store one and only one ExecutePolicy object that expires after some inactivity.
    // The side effect is that it may take this many minutes to update the policy that is cached.
    private LoadingCache<String, ExecutePolicy> theExecutionPolicy = CacheBuilder.newBuilder()
            .expireAfterWrite(1, TimeUnit.MINUTES).removalListener(new RemovalListener<String, ExecutePolicy>() {
                @Override
                public void onRemoval(RemovalNotification<String, ExecutePolicy> arg) {
                    arg.getValue().close();
                }
            }).build(new CacheLoader<String, ExecutePolicy>() {
                public ExecutePolicy load(String key) throws IOException {
                    logger.info("Updating the cached execute policy");
                    return new ExecutePolicy(DefaultConfigService.this);
                }
            });

    private ServletContext servletContext;

    protected DefaultConfigService() {
        // Only the unit tests config service uses this constructor.
    }

    @Override
    public void initialize(ServletContext sce) throws ConfigException {
        this.servletContext = sce;
        String contextPath = sce.getContextPath();
        logger.info("DefaultConfigService was created with a servlet context " + contextPath);

        try {
            String pathToVersionTxt = sce.getRealPath("ui/comm/version.txt");
            logger.debug("The full path to the version.txt is " + pathToVersionTxt);
            List<String> lines = Files.readAllLines(Paths.get(pathToVersionTxt), Charset.forName("UTF-8"));
            for (String line : lines) {
                configlogger.info(line);
            }
        } catch (Throwable t) {
            logger.fatal("Unable to determine appliance version", t);
        }

        try {
            // We first try Java system properties for this appliance's identity
            // If a property is not defined, then we check the environment.
            // This gives us the ability to cater to unit tests as well as running using buildAndDeploy scripts without touching the server.xml file.
            // Probably not the most standard way but suited to this need.
            // Finally, we use the local machine's hostname as the myidentity.
            myIdentity = System.getProperty(ARCHAPPL_MYIDENTITY);
            if (myIdentity == null) {
                myIdentity = System.getenv(ARCHAPPL_MYIDENTITY);
                if (myIdentity != null) {
                    logger.info("Obtained my identity from environment variable " + myIdentity);
                } else {
                    logger.info("Using the local machine's hostname " + myIdentity + " as my identity");
                    myIdentity = InetAddress.getLocalHost().getCanonicalHostName();
                }
                if (myIdentity == null) {
                    throw new ConfigException("Unable to determine identity of this appliance");
                }
            } else {
                logger.info("Obtained my identity from Java system properties " + myIdentity);
            }

            logger.info("My identity is " + myIdentity);
        } catch (Exception ex) {
            String msg = "Cannot determine this appliance's identity using either the environment variable "
                    + ARCHAPPL_MYIDENTITY + " or the java system property " + ARCHAPPL_MYIDENTITY;
            configlogger.fatal(msg);
            throw new ConfigException(msg, ex);
        }
        // Appliances should be local and come straight from persistence.
        try {
            appliances = AppliancesList.loadAppliancesXML(servletContext);
        } catch (Exception ex) {
            throw new ConfigException("Exception loading appliances.xml", ex);
        }

        myApplianceInfo = appliances.get(myIdentity);
        if (myApplianceInfo == null)
            throw new ConfigException("Unable to determine applianceinfo using identity " + myIdentity);
        configlogger.info("My identity is " + myApplianceInfo.getIdentity() + " and my mgmt URL is "
                + myApplianceInfo.getMgmtURL());

        // To make sure we are not starting multiple appliance with the same identity, we make sure that the hostnames match
        try {
            String machineHostName = InetAddress.getLocalHost().getCanonicalHostName();
            String[] myAddrParts = myApplianceInfo.getClusterInetPort().split(":");
            String myHostNameFromInfo = myAddrParts[0];
            if (myHostNameFromInfo.equals("localhost")) {
                logger.debug(
                        "Using localhost for the cluster inet port. If you are indeed running a cluster, the cluster members will not join the cluster.");
            } else if (myHostNameFromInfo.equals(machineHostName)) {
                logger.debug(
                        "Hostname from config and hostname from InetAddress match exactly; we are correctly configured "
                                + machineHostName);
            } else if (InetAddressValidator.getInstance().isValid(myHostNameFromInfo)) {
                logger.debug("Using ipAddress for cluster config " + myHostNameFromInfo);
            } else {
                String msg = "The hostname from appliances.xml is " + myHostNameFromInfo
                        + " and from a call to InetAddress.getLocalHost().getCanonicalHostName() (typially FQDN) is "
                        + machineHostName
                        + ". These are not identical. They are probably equivalent but to prevent multiple appliances binding to the same identity we enforce this equality.";
                configlogger.fatal(msg);
                throw new ConfigException(msg);
            }
        } catch (UnknownHostException ex) {
            configlogger.error(
                    "Got an UnknownHostException when trying to determine the hostname. This happens when DNS is not set correctly on this machine (for example, when using VM's. See the documentation for InetAddress.getLocalHost().getCanonicalHostName()");
        }

        try {
            String archApplPropertiesFileName = System.getProperty(ARCHAPPL_PROPERTIES_FILENAME);
            if (archApplPropertiesFileName == null) {
                archApplPropertiesFileName = System.getenv(ARCHAPPL_PROPERTIES_FILENAME);
            }
            if (archApplPropertiesFileName == null) {
                archApplPropertiesFileName = new URL(this.getClass().getClassLoader()
                        .getResource(DEFAULT_ARCHAPPL_PROPERTIES_FILENAME).toString()).getPath();
                configlogger.info(
                        "Loading archappl.properties from the webapp classpath " + archApplPropertiesFileName);
            } else {
                configlogger.info("Loading archappl.properties using the environment/JVM property from "
                        + archApplPropertiesFileName);
            }
            try (InputStream is = new FileInputStream(new File(archApplPropertiesFileName))) {
                archapplproperties.load(is);
                configlogger.info(
                        "Done loading installation specific properties file from " + archApplPropertiesFileName);
            } catch (Exception ex) {
                throw new ConfigException(
                        "Exception loading installation specific properties file " + archApplPropertiesFileName,
                        ex);
            }
        } catch (ConfigException cex) {
            throw cex;
        } catch (Exception ex) {
            configlogger.fatal("Exception loading the appliance properties file", ex);
        }

        switch (contextPath) {
        case "/mgmt":
            warFile = WAR_FILE.MGMT;
            this.mgmtRuntime = new MgmtRuntimeState(this);
            break;
        case "/engine":
            warFile = WAR_FILE.ENGINE;
            this.engineContext = new EngineContext(this);
            break;
        case "/retrieval":
            warFile = WAR_FILE.RETRIEVAL;
            this.retrievalState = new RetrievalState(this);
            break;
        case "/etl":
            this.etlPVLookup = new PBThreeTierETLPVLookup(this);
            warFile = WAR_FILE.ETL;
            break;
        default:
            logger.error("We seem to have introduced a new component into the system " + contextPath);
        }

        String pvName2KeyMappingClass = this.getInstallationProperties()
                .getProperty(ARCHAPPL_PVNAME_TO_KEY_MAPPING_CLASSNAME);
        if (pvName2KeyMappingClass == null || pvName2KeyMappingClass.equals("")
                || pvName2KeyMappingClass.length() < 1) {
            logger.info("Using the default key mapping class");
            pvName2KeyConverter = new ConvertPVNameToKey();
            pvName2KeyConverter.initialize(this);
        } else {
            try {
                logger.info("Using " + pvName2KeyMappingClass + " as the name to key mapping class");
                pvName2KeyConverter = (PVNameToKeyMapping) Class.forName(pvName2KeyMappingClass).newInstance();
                pvName2KeyConverter.initialize(this);
            } catch (Exception ex) {
                logger.fatal("Cannot initialize pv name to key mapping class " + pvName2KeyMappingClass, ex);
                throw new ConfigException(
                        "Cannot initialize pv name to key mapping class " + pvName2KeyMappingClass, ex);
            }
        }

        String runtimeFieldsListStr = this.getInstallationProperties()
                .getProperty("org.epics.archiverappliance.config.RuntimeKeys");
        if (runtimeFieldsListStr != null && !runtimeFieldsListStr.isEmpty()) {
            logger.debug("Got runtime fields from the properties file " + runtimeFieldsListStr);
            String[] runTimeFieldsArr = runtimeFieldsListStr.split(",");
            for (String rf : runTimeFieldsArr) {
                this.runTimeFields.add(rf.trim());
            }
        }

        startupExecutor = Executors.newScheduledThreadPool(1, new ThreadFactory() {
            @Override
            public Thread newThread(Runnable r) {
                Thread t = new Thread(r);
                t.setName("Startup executor");
                return t;
            }
        });

        this.addShutdownHook(new Runnable() {
            @Override
            public void run() {
                logger.info("Shutting down startup scheduled executor...");
                startupExecutor.shutdown();
            }
        });

        this.startupState = STARTUP_SEQUENCE.READY_TO_JOIN_APPLIANCE;
        if (this.warFile == WAR_FILE.MGMT) {
            logger.info("Scheduling webappReady's for the mgmt webapp ");
            MgmtPostStartup mgmtPostStartup = new MgmtPostStartup(this);
            ScheduledFuture<?> postStartupFuture = startupExecutor.scheduleAtFixedRate(mgmtPostStartup, 10, 20,
                    TimeUnit.SECONDS);
            mgmtPostStartup.setCancellingFuture(postStartupFuture);
        } else {
            logger.info("Scheduling webappReady's for the non-mgmt webapp " + this.warFile.toString());
            NonMgmtPostStartup nonMgmtPostStartup = new NonMgmtPostStartup(this, this.warFile.toString());
            ScheduledFuture<?> postStartupFuture = startupExecutor.scheduleAtFixedRate(nonMgmtPostStartup, 10, 20,
                    TimeUnit.SECONDS);
            nonMgmtPostStartup.setCancellingFuture(postStartupFuture);
        }

        // Measure some JMX metrics once a minute
        startupExecutor.scheduleAtFixedRate(new Runnable() {
            @Override
            public void run() {
                processMetrics.takeMeasurement();
            }
        }, 60, 60, TimeUnit.SECONDS);
    }

    /* (non-Javadoc)
     * @see org.epics.archiverappliance.config.ConfigService#postStartup()
     */
    @Override
    public void postStartup() throws ConfigException {
        if (this.startupState != STARTUP_SEQUENCE.READY_TO_JOIN_APPLIANCE) {
            configlogger.info("Webapp is not in correct state for postStartup " + this.getWarFile().toString()
                    + ". It is in " + this.startupState.toString());
            return;
        }

        this.startupState = STARTUP_SEQUENCE.POST_STARTUP_RUNNING;
        configlogger.info("Post startup for " + this.getWarFile().toString());

        // Inherit logging from log4j configuration.
        try {
            PlatformLoggingMXBean logging = ManagementFactory.getPlatformMXBean(PlatformLoggingMXBean.class);
            if (logging != null) {
                java.util.logging.Logger.getLogger("com.hazelcast");
                if (clusterLogger.isDebugEnabled()) {
                    logging.setLoggerLevel("com.hazelcast", java.util.logging.Level.FINE.toString());
                } else if (clusterLogger.isInfoEnabled()) {
                    logging.setLoggerLevel("com.hazelcast", java.util.logging.Level.INFO.toString());
                } else {
                    logger.info(
                            "Setting clustering logging based on log levels for cluster." + getClass().getName());
                    logging.setLoggerLevel("com.hazelcast", java.util.logging.Level.SEVERE.toString());
                }
            }

            Logger hzMain = Logger.getLogger("com.hazelcast");
            if (clusterLogger.isDebugEnabled()) {
                hzMain.setLevel(Level.DEBUG);
            } else if (clusterLogger.isInfoEnabled()) {
                hzMain.setLevel(Level.INFO);
            } else {
                logger.info("Setting clustering logging based on log levels for cluster." + getClass().getName());
                hzMain.setLevel(Level.FATAL);
            }
        } catch (Exception ex) {
            logger.error("Exception setting logging JVM levels ", ex);
        }

        // Add this to the system props before doing anything with Hz
        System.getProperties().put("hazelcast.logging.type", "log4j");

        HazelcastInstance hzinstance = null;

        // Set the thread count to control how may threads this library spawns.
        Properties hzThreadCounts = new Properties();
        if (System.getenv().containsKey("ARCHAPPL_ALL_APPS_ON_ONE_JVM")) {
            logger.info("Reducing the generic clustering thread counts.");
            hzThreadCounts.put("hazelcast.clientengine.thread.count", "2");
            hzThreadCounts.put("hazelcast.operation.generic.thread.count", "2");
            hzThreadCounts.put("hazelcast.operation.thread.count", "2");
        }

        if (this.warFile == WAR_FILE.MGMT) {
            // The management webapps are the head honchos in the cluster. We set them up differently

            configlogger.debug("Initializing the MGMT webapp's clustering");
            // If we have a hazelcast.xml in the servlet classpath, the XmlConfigBuilder picks that up.
            // If not we use the default config found in hazelcast.jar
            // We then alter this config to suit our purposes.
            Config config = new XmlConfigBuilder().build();
            try {
                if (this.getClass().getResource("hazelcast.xml") == null) {
                    logger.info("We override the default cluster config by disabling multicast discovery etc.");
                    // We do not use multicast as it is not supported on all networks.
                    config.getNetworkConfig().getJoin().getMulticastConfig().setEnabled(false);
                    // We use TCPIP to discover the members in the cluster.
                    // This is part of the config that comes from appliance.xml
                    config.getNetworkConfig().getJoin().getTcpIpConfig().setEnabled(true);
                    // Clear any tcpip config that comes from the default config
                    // This gets rid of the localhost in the default that prevents clusters from forming..
                    // If we need localhost, we'll add it back later.
                    config.getNetworkConfig().getJoin().getTcpIpConfig().clear();
                    // Enable interfaces; we seem to need this after 2.4 for clients to work correctly in a multi-homed environment.
                    // We'll add the actual interface later below
                    config.getNetworkConfig().getInterfaces().setEnabled(true);
                    config.getNetworkConfig().getInterfaces().clear();

                    // We don't really use the authentication provided by the tool; however, we set it to some default
                    config.getGroupConfig().setName("archappl");
                    config.getGroupConfig().setPassword("archappl");

                    // Backup count is 1 by default; we set it explicitly however...
                    config.getMapConfig("default").setBackupCount(1);

                    config.setProperty("hazelcast.logging.type", "log4j");
                } else {
                    logger.debug(
                            "There is a hazelcast.xml in the classpath; skipping default configuration in the code.");
                }
            } catch (Exception ex) {
                throw new ConfigException("Exception configuring cluster", ex);
            }

            config.setInstanceName(myIdentity);

            if (!hzThreadCounts.isEmpty()) {
                logger.info("Reducing the generic clustering thread counts.");
                config.getProperties().putAll(hzThreadCounts);
            }

            try {
                String[] myAddrParts = myApplianceInfo.getClusterInetPort().split(":");
                String myHostName = myAddrParts[0];
                InetAddress myInetAddr = InetAddress.getByName(myHostName);
                if (!myHostName.equals("localhost") && myInetAddr.isLoopbackAddress()) {
                    logger.info("Address for this appliance -- " + myInetAddr.toString()
                            + " is a loopback address. Changing this to 127.0.0.1 to clustering happy");
                    myInetAddr = InetAddress.getByName("127.0.0.1");
                }
                int myClusterPort = Integer.parseInt(myAddrParts[1]);

                logger.debug("We do not let the port auto increment for the MGMT webap");
                config.getNetworkConfig().setPortAutoIncrement(false);

                config.getNetworkConfig().setPort(myClusterPort);
                config.getNetworkConfig().getInterfaces().addInterface(myInetAddr.getHostAddress());
                configlogger.info("Setting my cluster port base to " + myClusterPort + " and using interface "
                        + myInetAddr.getHostAddress());

                for (ApplianceInfo applInfo : appliances.values()) {
                    if (applInfo.getIdentity().equals(myIdentity) && this.warFile == WAR_FILE.MGMT) {
                        logger.debug("Not adding myself to the discovery process when I am the mgmt webapp");
                    } else {
                        String[] addressparts = applInfo.getClusterInetPort().split(":");
                        String inetaddrpart = addressparts[0];
                        try {
                            InetAddress inetaddr = InetAddress.getByName(inetaddrpart);
                            if (!inetaddrpart.equals("localhost") && inetaddr.isLoopbackAddress()) {
                                logger.info("Address for appliance " + applInfo.getIdentity() + " -  "
                                        + inetaddr.toString()
                                        + " is a loopback address. Changing this to 127.0.0.1 to clustering happy");
                                inetaddr = InetAddress.getByName("127.0.0.1");
                            }
                            int clusterPort = Integer.parseInt(addressparts[1]);
                            logger.info("Adding " + applInfo.getIdentity()
                                    + " from appliances.xml to the cluster discovery using cluster inetport "
                                    + inetaddr.toString() + ":" + clusterPort);
                            config.getNetworkConfig().getJoin().getTcpIpConfig()
                                    .addMember(inetaddr.getHostAddress() + ":" + clusterPort);
                        } catch (UnknownHostException ex) {
                            configlogger.info("Cannnot resolve the IP address for appliance " + inetaddrpart
                                    + ". Skipping adding this appliance to the cliuster.");
                        }
                    }
                }
                hzinstance = Hazelcast.newHazelcastInstance(config);
            } catch (Exception ex) {
                throw new ConfigException("Exception adding member to cluster", ex);
            }
        } else {
            // All other webapps are "native" clients.
            try {
                configlogger.debug("Initializing a non-mgmt webapp's clustering");
                ClientConfig clientConfig = new ClientConfig();
                clientConfig.getGroupConfig().setName("archappl");
                clientConfig.getGroupConfig().setPassword("archappl");
                clientConfig.setExecutorPoolSize(4);
                // Non mgmt client can only connect to their MGMT webapp.
                String[] myAddrParts = myApplianceInfo.getClusterInetPort().split(":");
                String myHostName = myAddrParts[0];
                InetAddress myInetAddr = InetAddress.getByName(myHostName);
                if (!myHostName.equals("localhost") && myInetAddr.isLoopbackAddress()) {
                    logger.info("Address for this appliance -- " + myInetAddr.toString()
                            + " is a loopback address. Changing this to 127.0.0.1 to clustering happy");
                    myInetAddr = InetAddress.getByName("127.0.0.1");
                }
                int myClusterPort = Integer.parseInt(myAddrParts[1]);

                configlogger.debug(this.warFile + " connecting as a native client to " + myInetAddr.getHostAddress()
                        + ":" + myClusterPort);
                clientConfig.getNetworkConfig().addAddress(myInetAddr.getHostAddress() + ":" + myClusterPort);
                clientConfig.setProperty("hazelcast.logging.type", "log4j");

                if (!hzThreadCounts.isEmpty()) {
                    logger.info("Reducing the generic clustering thread counts.");
                    clientConfig.getProperties().putAll(hzThreadCounts);
                }

                if (!clusterLogger.isDebugEnabled()) {
                    // The client code logs some SEVERE exceptions on shutdown when deploying on the same Tomcat container.
                    // These exceptions are confusing; ideally, we would not have to set the log levels like so.
                    Logger.getLogger("com.hazelcast.client.spi.impl.ClusterListenerThread").setLevel(Level.OFF);
                    Logger.getLogger("com.hazelcast.client.spi.ClientPartitionService").setLevel(Level.OFF);
                }
                hzinstance = HazelcastClient.newHazelcastClient(clientConfig);
            } catch (Exception ex) {
                throw new ConfigException("Exception adding client to cluster", ex);
            }
        }

        pv2appliancemapping = hzinstance.getMap("pv2appliancemapping");
        namedFlags = hzinstance.getMap("namedflags");
        typeInfos = hzinstance.getMap("typeinfo");
        archivePVRequests = hzinstance.getMap("archivePVRequests");
        channelArchiverDataServers = hzinstance.getMap("channelArchiverDataServers");
        clusterInet2ApplianceIdentity = hzinstance.getMap("clusterInet2ApplianceIdentity");
        aliasNamesToRealNames = hzinstance.getMap("aliasNamesToRealNames");
        pv2ChannelArchiverDataServer = hzinstance.getMap("pv2ChannelArchiverDataServer");
        pubSub = hzinstance.getTopic("pubSub");

        final HazelcastInstance shutdownHzInstance = hzinstance;
        shutdownHooks.add(0, new Runnable() {
            @Override
            public void run() {
                logger.debug("Shutting down clustering instance in webapp " + warFile.toString());
                shutdownHzInstance.shutdown();
            }
        });

        if (this.warFile == WAR_FILE.MGMT) {
            Cluster cluster = hzinstance.getCluster();
            String localInetPort = getMemberKey(cluster.getLocalMember());
            clusterInet2ApplianceIdentity.put(localInetPort, myIdentity);
            logger.debug("Adding myself " + myIdentity + " as having inetport " + localInetPort);
            hzinstance.getMap("clusterInet2ApplianceIdentity")
                    .addEntryListener(new EntryAddedListener<Object, Object>() {
                        @Override
                        public void entryAdded(EntryEvent<Object, Object> event) {
                            String appliden = (String) event.getValue();
                            appliancesInCluster.add(appliden);
                            logger.info("Adding appliance " + appliden
                                    + " to the list of active appliances as inetport " + ((String) event.getKey()));
                        }
                    }, true);
            hzinstance.getMap("clusterInet2ApplianceIdentity")
                    .addEntryListener(new EntryRemovedListener<Object, Object>() {
                        @Override
                        public void entryRemoved(EntryEvent<Object, Object> event) {
                            String appliden = (String) event.getValue();
                            appliancesInCluster.remove(appliden);
                            logger.info("Removing appliance " + appliden
                                    + " from the list of active appliancesas inetport "
                                    + ((String) event.getKey()));
                        }
                    }, true);

            logger.debug(
                    "Establishing a cluster membership listener to detect when appliances drop off the cluster");
            cluster.addMembershipListener(new MembershipListener() {
                public void memberAdded(MembershipEvent membersipEvent) {
                    Member member = membersipEvent.getMember();
                    String inetPort = getMemberKey(member);
                    if (clusterInet2ApplianceIdentity.containsKey(inetPort)) {
                        String appliden = clusterInet2ApplianceIdentity.get(inetPort);
                        appliancesInCluster.add(appliden);
                        configlogger.info("Adding newly started appliance " + appliden
                                + " to the list of active appliances for inetport " + inetPort);
                    } else {
                        logger.debug("Skipping adding appliance using inetport " + inetPort
                                + " to the list of active instances as we do not have a mapping to its identity");
                    }
                }

                public void memberRemoved(MembershipEvent membersipEvent) {
                    Member member = membersipEvent.getMember();
                    String inetPort = getMemberKey(member);
                    if (clusterInet2ApplianceIdentity.containsKey(inetPort)) {
                        String appliden = clusterInet2ApplianceIdentity.get(inetPort);
                        appliancesInCluster.remove(appliden);
                        configlogger.info("Removing appliance " + appliden + " from the list of active appliances");
                    } else {
                        configlogger.debug("Received member removed event for " + inetPort);
                    }
                }

                @Override
                public void memberAttributeChanged(MemberAttributeEvent membersipEvent) {
                    Member member = membersipEvent.getMember();
                    String inetPort = getMemberKey(member);
                    configlogger.debug("Received membership attribute changed event for " + inetPort);
                }
            });

            logger.debug(
                    "Adding the current members in the cluster after establishing the cluster membership listener");
            for (Member member : cluster.getMembers()) {
                String mbrInetPort = getMemberKey(member);
                logger.debug("Found member " + mbrInetPort);
                if (clusterInet2ApplianceIdentity.containsKey(mbrInetPort)) {
                    String appliden = clusterInet2ApplianceIdentity.get(mbrInetPort);
                    appliancesInCluster.add(appliden);
                    logger.info("Adding appliance " + appliden + " to the list of active appliances for inetport "
                            + mbrInetPort);
                } else {
                    logger.debug("Skipping adding appliance using inetport " + mbrInetPort
                            + " to the list of active instances as we do not have a mapping to its identity");
                }
            }
            logger.info("Established subscription(s) for appliance availability");

            if (this.getInstallationProperties().containsKey(ARCHAPPL_NAMEDFLAGS_PROPERTIES_FILE_PROPERTY)) {
                String namedFlagsFileName = (String) this.getInstallationProperties()
                        .get(ARCHAPPL_NAMEDFLAGS_PROPERTIES_FILE_PROPERTY);
                configlogger.info("Loading named flags from file " + namedFlagsFileName);
                File namedFlagsFile = new File(namedFlagsFileName);
                if (!namedFlagsFile.exists()) {
                    configlogger.error(
                            "File containing named flags " + namedFlagsFileName + " specified but not present");
                } else {
                    Properties namedFlagsFromFile = new Properties();
                    try (FileInputStream is = new FileInputStream(namedFlagsFile)) {
                        namedFlagsFromFile.load(is);
                        for (Object namedFlagFromFile : namedFlagsFromFile.keySet()) {
                            try {
                                String namedFlagFromFileStr = (String) namedFlagFromFile;
                                Boolean namedFlagFromFileValue = Boolean
                                        .parseBoolean((String) namedFlagsFromFile.get(namedFlagFromFileStr));
                                logger.debug("Setting named flag " + namedFlagFromFileStr + " to "
                                        + namedFlagFromFileValue);
                                this.namedFlags.put(namedFlagFromFileStr, namedFlagFromFileValue);
                            } catch (Exception ex) {
                                logger.error("Exception loading named flag from file" + namedFlagsFileName, ex);
                            }
                        }
                    } catch (Exception ex) {
                        configlogger.error("Exception loading named flags from " + namedFlagsFileName, ex);
                    }
                }
            }
        }

        if (this.warFile == WAR_FILE.ENGINE) {
            // It can take a while for the engine to start up.
            // We probably want to do this in the background so that the appliance as a whole starts up quickly and we get retrieval up and running quickly.
            this.startupExecutor.schedule(new Runnable() {
                @Override
                public void run() {
                    try {
                        logger.debug("Starting up the engine's channels on startup.");
                        archivePVSonStartup();
                        logger.debug("Done starting up the engine's channels in startup.");
                    } catch (Throwable t) {
                        configlogger.fatal("Exception starting up the engine channels on startup", t);
                    }
                }
            }, 1, TimeUnit.SECONDS);
        } else if (this.warFile == WAR_FILE.ETL) {
            this.etlPVLookup.postStartup();
        } else if (this.warFile == WAR_FILE.MGMT) {
            pvsForThisAppliance = new ConcurrentSkipListSet<String>();
            pausedPVsForThisAppliance = new ConcurrentSkipListSet<String>();

            initializePersistenceLayer();

            loadTypeInfosFromPersistence();

            loadAliasesFromPersistence();

            loadArchiveRequestsFromPersistence();

            loadExternalServersFromPersistence();

            registerForNewExternalServers(hzinstance.getMap("channelArchiverDataServers"));

            // Cache the aggregate of all the PVs that are registered to this appliance.
            logger.debug("Building a local aggregate of PV infos that are registered to this appliance");
            for (String pvName : getPVsForThisAppliance()) {
                if (!pvsForThisAppliance.contains(pvName)) {
                    applianceAggregateInfo.addInfoForPV(pvName, this.getTypeInfoForPV(pvName), this);
                }
            }
        }

        // Register for changes to the typeinfo map.
        logger.info("Registering for changes to typeinfos");
        hzinstance.getMap("typeinfo").addEntryListener(new EntryAddedListener<Object, Object>() {
            @Override
            public void entryAdded(EntryEvent<Object, Object> entryEvent) {
                logger.debug("Received entryAdded for pvTypeInfo");
                PVTypeInfo typeInfo = (PVTypeInfo) entryEvent.getValue();
                String pvName = typeInfo.getPvName();
                eventBus.post(new PVTypeInfoEvent(pvName, typeInfo, ChangeType.TYPEINFO_ADDED));
                if (persistanceLayer != null) {
                    try {
                        persistanceLayer.putTypeInfo(pvName, typeInfo);
                    } catch (Exception ex) {
                        logger.error("Exception persisting pvTypeInfo for pv " + pvName, ex);
                    }
                }
            }
        }, true);
        hzinstance.getMap("typeinfo").addEntryListener(new EntryRemovedListener<Object, Object>() {
            @Override
            public void entryRemoved(EntryEvent<Object, Object> entryEvent) {
                PVTypeInfo typeInfo = (PVTypeInfo) entryEvent.getOldValue();
                String pvName = typeInfo.getPvName();
                logger.info("Received entryRemoved for pvTypeInfo " + pvName);
                eventBus.post(new PVTypeInfoEvent(pvName, typeInfo, ChangeType.TYPEINFO_DELETED));
                if (persistanceLayer != null) {
                    try {
                        persistanceLayer.deleteTypeInfo(pvName);
                    } catch (Exception ex) {
                        logger.error("Exception deleting pvTypeInfo for pv " + pvName, ex);
                    }
                }
            }
        }, true);
        hzinstance.getMap("typeinfo").addEntryListener(new EntryUpdatedListener<Object, Object>() {
            @Override
            public void entryUpdated(EntryEvent<Object, Object> entryEvent) {
                PVTypeInfo typeInfo = (PVTypeInfo) entryEvent.getValue();
                String pvName = typeInfo.getPvName();
                eventBus.post(new PVTypeInfoEvent(pvName, typeInfo, ChangeType.TYPEINFO_MODIFIED));
                logger.debug("Received entryUpdated for pvTypeInfo");
                if (persistanceLayer != null) {
                    try {
                        persistanceLayer.putTypeInfo(pvName, typeInfo);
                    } catch (Exception ex) {
                        logger.error("Exception persisting pvTypeInfo for pv " + pvName, ex);
                    }
                }
            }
        }, true);

        eventBus.register(this);

        pubSub.addMessageListener(new MessageListener<PubSubEvent>() {
            @Override
            public void onMessage(Message<PubSubEvent> pubSubEventMsg) {
                PubSubEvent pubSubEvent = pubSubEventMsg.getMessageObject();
                if (pubSubEvent.getDestination() != null) {
                    if (pubSubEvent.getDestination().equals("ALL")
                            || (pubSubEvent.getDestination().startsWith(myIdentity) && pubSubEvent.getDestination()
                                    .endsWith(DefaultConfigService.this.warFile.toString()))) {
                        // We publish messages from hazelcast into this VM only if the intened WAR file is us.
                        logger.debug("Publishing event into this JVM " + pubSubEvent.generateEventDescription());
                        // In this case, we set the source as being the cluster to prevent republishing back into the cluster.
                        pubSubEvent.markSourceAsCluster();
                        eventBus.post(pubSubEvent);
                    } else {
                        logger.debug("Skipping publishing event into this JVM "
                                + pubSubEvent.generateEventDescription() + " as destination is not me "
                                + DefaultConfigService.this.warFile.toString());
                    }
                } else {
                    logger.debug("Skipping publishing event with null destination");
                }
            }
        });

        logger.info("Done registering for changes to typeinfos");

        this.startupState = STARTUP_SEQUENCE.STARTUP_COMPLETE;
        configlogger.info("Start complete for webapp " + this.warFile);
    }

    @Override
    public STARTUP_SEQUENCE getStartupState() {
        return this.startupState;
    }

    @Subscribe
    public void updatePVSForThisAppliance(PVTypeInfoEvent event) {
        if (logger.isDebugEnabled())
            logger.debug("Received pvTypeInfo change event for pv " + event.getPvName());
        PVTypeInfo typeInfo = event.getTypeInfo();
        String pvName = typeInfo.getPvName();
        if (typeInfo.getApplianceIdentity().equals(myApplianceInfo.getIdentity())) {
            if (event.getChangeType() == ChangeType.TYPEINFO_DELETED) {
                if (pvsForThisAppliance != null) {
                    if (pvsForThisAppliance.contains(pvName)) {
                        logger.debug("Removing pv " + pvName
                                + " from the locally cached copy of pvs for this appliance");
                        pvsForThisAppliance.remove(pvName);
                        pausedPVsForThisAppliance.remove(pvName);
                        // For now, we do not anticipate many PVs being deleted from the cache to worry about keeping applianceAggregateInfo upto date...
                        // This may change later... 
                        String[] parts = this.pvName2KeyConverter.breakIntoParts(pvName);
                        for (String part : parts) {
                            parts2PVNamesForThisAppliance.get(part).remove(pvName);
                        }
                    }
                }
            } else {
                if (pvsForThisAppliance != null) {
                    if (!pvsForThisAppliance.contains(pvName)) {
                        logger.debug(
                                "Adding pv " + pvName + " to the locally cached copy of pvs for this appliance");
                        pvsForThisAppliance.add(pvName);
                        if (typeInfo.isPaused()) {
                            pausedPVsForThisAppliance.add(typeInfo.getPvName());
                        }
                        String[] parts = this.pvName2KeyConverter.breakIntoParts(pvName);
                        for (String part : parts) {
                            if (!parts2PVNamesForThisAppliance.containsKey(part)) {
                                parts2PVNamesForThisAppliance.put(part, new ConcurrentSkipListSet<String>());
                            }
                            parts2PVNamesForThisAppliance.get(part).add(pvName);
                        }
                        applianceAggregateInfo.addInfoForPV(pvName, typeInfo, this);
                    } else {
                        if (typeInfo.isPaused()) {
                            pausedPVsForThisAppliance.add(typeInfo.getPvName());
                        } else {
                            pausedPVsForThisAppliance.remove(typeInfo.getPvName());
                        }
                    }
                }
            }
        }
    }

    @Subscribe
    public void publishEventIntoCluster(PubSubEvent pubSubEvent) {
        if (pubSubEvent.isSourceCluster()) {
            logger.debug("Skipping publishing events from the cluster back into the cluster "
                    + pubSubEvent.generateEventDescription());
            return;
        }

        if (pubSubEvent.getDestination().startsWith(myIdentity)
                && pubSubEvent.getDestination().endsWith(this.warFile.toString())) {
            logger.debug(this.warFile + " - Skipping publishing event " + pubSubEvent.generateEventDescription()
                    + " meant for myself " + this.warFile.toString());
        } else {
            pubSubEvent.setSource(myIdentity);
            logger.debug(this.warFile + " - Publishing event from local event bus onto cluster "
                    + pubSubEvent.generateEventDescription());
            pubSub.publish(pubSubEvent);
        }
    }

    /**
     * Get the PVs that belong to this appliance and start archiving them
     * Needless to day, this gets done only in the engine.
     */
    private void archivePVSonStartup() {
        configlogger.debug("Start archiving PVs from persistence.");
        int secondsToBuffer = PVTypeInfo.getSecondsToBuffer(this);
        // To prevent broadcast storms, we pause for pausePerGroup seconds for every pausePerGroup PVs
        int currentPVCount = 0;
        int pausePerGroupPVCount = 2000;
        int pausePerGroupPauseTimeInSeconds = 2;
        for (String pvName : this.getPVsForThisAppliance()) {
            try {
                PVTypeInfo typeInfo = typeInfos.get(pvName);
                if (typeInfo == null) {
                    logger.error("On restart, cannot find typeinfo for pv " + pvName + ". Not archiving");
                    continue;
                }

                if (typeInfo.isPaused()) {
                    logger.debug("Skipping archiving paused PV " + pvName + " on startup");
                    continue;
                }

                ArchDBRTypes dbrType = typeInfo.getDBRType();
                float samplingPeriod = typeInfo.getSamplingPeriod();
                SamplingMethod samplingMethod = typeInfo.getSamplingMethod();
                StoragePlugin firstDest = StoragePluginURLParser.parseStoragePlugin(typeInfo.getDataStores()[0],
                        this);

                Timestamp lastKnownTimestamp = typeInfo.determineLastKnownEventFromStores(this);
                if (logger.isDebugEnabled())
                    logger.debug("Last known timestamp from ETL stores is for pv " + pvName + " is "
                            + TimeUtils.convertToHumanReadableString(lastKnownTimestamp));

                ArchiveEngine.archivePV(pvName, samplingPeriod, samplingMethod, secondsToBuffer, firstDest, this,
                        dbrType, lastKnownTimestamp, typeInfo.getControllingPV(), typeInfo.getArchiveFields(),
                        typeInfo.getHostName(), typeInfo.isUsePVAccess(), typeInfo.isUseDBEProperties());
                currentPVCount++;
                if (currentPVCount % pausePerGroupPVCount == 0) {
                    logger.debug(
                            "Sleeping for " + pausePerGroupPauseTimeInSeconds + " to prevent CA search storms");
                    Thread.sleep(pausePerGroupPauseTimeInSeconds * 1000);
                }
            } catch (Throwable t) {
                logger.error("Exception starting up archiving of PV " + pvName + ". Moving on to the next pv.", t);
            }
        }
        configlogger.debug("Started " + currentPVCount + " PVs from persistence.");
    }

    @Override
    public boolean isStartupComplete() {
        return startupState == STARTUP_SEQUENCE.STARTUP_COMPLETE;
    }

    @Override
    public Properties getInstallationProperties() {
        return archapplproperties;
    }

    @Override
    public Collection<ApplianceInfo> getAppliancesInCluster() {
        ArrayList<ApplianceInfo> sortedAppliances = new ArrayList<ApplianceInfo>();
        for (ApplianceInfo info : appliances.values()) {
            if (appliancesInCluster.contains(info.getIdentity())) {
                sortedAppliances.add(info);
            } else {
                logger.debug("Skipping appliance that is in the persistence but not in the cluster"
                        + info.getIdentity());
            }
        }

        Collections.sort(sortedAppliances, new Comparator<ApplianceInfo>() {
            @Override
            public int compare(ApplianceInfo o1, ApplianceInfo o2) {
                return o1.getIdentity().compareTo(o2.getIdentity());
            }
        });
        return sortedAppliances;
    }

    @Override
    public ApplianceInfo getMyApplianceInfo() {
        return myApplianceInfo;
    }

    @Override
    public ApplianceInfo getAppliance(String identity) {
        return appliances.get(identity);
    }

    @Override
    public Collection<String> getAllPVs() {
        List<PVApplianceCombo> sortedCombos = getSortedPVApplianceCombo();
        ArrayList<String> allPVs = new ArrayList<String>();
        for (PVApplianceCombo combo : sortedCombos) {
            allPVs.add(combo.pvName);
        }
        return allPVs;
    }

    @Override
    public ApplianceInfo getApplianceForPV(String pvName) {
        ApplianceInfo applianceInfo = pv2appliancemapping.get(pvName);
        if (applianceInfo == null && this.persistanceLayer != null) {
            try {
                PVTypeInfo typeInfo = this.persistanceLayer.getTypeInfo(pvName);
                if (typeInfo != null) {
                    applianceInfo = this.getAppliance(typeInfo.getApplianceIdentity());
                }
            } catch (IOException ex) {
                logger.error("Exception lookin up appliance for pv in persistence", ex);
            }
        }
        return applianceInfo;
    }

    @Override
    public Iterable<String> getPVsForAppliance(ApplianceInfo info) {
        String identity = info.getIdentity();
        List<PVApplianceCombo> sortedCombos = getSortedPVApplianceCombo();
        ArrayList<String> pvsForAppliance = new ArrayList<String>();
        for (PVApplianceCombo combo : sortedCombos) {
            if (combo.applianceIdentity.equals(identity)) {
                pvsForAppliance.add(combo.pvName);
            }
        }
        return pvsForAppliance;
    }

    @Override
    public Iterable<String> getPVsForThisAppliance() {
        if (pvsForThisAppliance != null) {
            logger.debug("Returning the locally cached copy of the pvs for this appliance");
            return pvsForThisAppliance;
        } else {
            logger.debug("Fetching the list of PVs for this appliance from the mgmt app");
            JSONArray pvs = GetUrlContent
                    .getURLContentAsJSONArray(myApplianceInfo.getMgmtURL() + "/getPVsForThisAppliance");
            LinkedList<String> retval = new LinkedList<String>();
            for (Object pv : pvs) {
                retval.add((String) pv);
            }
            return retval;
        }
    }

    @Override
    public boolean isBeingArchivedOnThisAppliance(String pvName) {
        boolean isField = PVNames.isField(pvName);
        String plainPVName = PVNames.stripFieldNameFromPVName(pvName);
        if (this.pvsForThisAppliance.contains(plainPVName)) {
            if (isField) {
                PVTypeInfo typeInfo = this.getTypeInfoForPV(plainPVName);
                if (typeInfo != null
                        && Arrays.asList(typeInfo.getArchiveFields()).contains(PVNames.getFieldName(pvName))) {
                    return true;
                }
            } else {
                return true;
            }
        }

        if (this.aliasNamesToRealNames.containsKey(plainPVName)) {
            plainPVName = this.aliasNamesToRealNames.get(plainPVName);

            if (this.pvsForThisAppliance.contains(plainPVName)) {
                if (isField) {
                    PVTypeInfo typeInfo = this.getTypeInfoForPV(plainPVName);
                    if (typeInfo != null
                            && Arrays.asList(typeInfo.getArchiveFields()).contains(PVNames.getFieldName(pvName))) {
                        return true;
                    }
                } else {
                    return true;
                }
            }
        }

        return false;
    }

    @Override
    public Set<String> getPVsForApplianceMatchingRegex(String nameToMatch) {
        logger.debug("Finding matching names for " + nameToMatch);
        LinkedList<String> fixedStringParts = new LinkedList<String>();
        String[] parts = this.pvName2KeyConverter.breakIntoParts(nameToMatch);
        Pattern fixedStringParttern = Pattern.compile("[a-zA-Z_0-9-]+");
        for (String part : parts) {
            if (fixedStringParttern.matcher(part).matches()) {
                logger.debug("Fixed string part " + part);
                fixedStringParts.add(part);
            } else {
                logger.debug("Regex string part " + part);
            }
        }

        if (fixedStringParts.size() > 0) {
            HashSet<String> ret = new HashSet<String>();
            HashSet<String> namesSubset = new HashSet<String>();
            // This reverse is probably specific to SLAC's namespace rules but it does make a big difference. 
            // Perhaps we can use a more intelligent way of choosing the specific path thru the trie.
            Collections.reverse(fixedStringParts);
            for (String fixedStringPart : fixedStringParts) {
                ConcurrentSkipListSet<String> pvNamesForPart = parts2PVNamesForThisAppliance.get(fixedStringPart);
                if (pvNamesForPart != null) {
                    if (namesSubset.isEmpty()) {
                        namesSubset.addAll(pvNamesForPart);
                    } else {
                        namesSubset.retainAll(pvNamesForPart);
                    }
                }
            }
            logger.debug("Using fixed string path matching against names " + namesSubset.size());
            Pattern pattern = Pattern.compile(nameToMatch);
            for (String pvName : namesSubset) {
                if (pattern.matcher(pvName).matches()) {
                    ret.add(pvName);
                }
            }
            return ret;
        } else {
            // The use pattern did not have any fixed elements at all. 
            // In this case we do brute force matching; should take longer.
            // This is also not optimal but probably don't want yet another list of PV's
            Pattern pattern = Pattern.compile(nameToMatch);
            HashSet<String> allNames = new HashSet<String>();
            HashSet<String> ret = new HashSet<String>();
            logger.debug("Using brute force pattern matching against names");
            for (ConcurrentSkipListSet<String> pvNamesForPart : parts2PVNamesForThisAppliance.values()) {
                allNames.addAll(pvNamesForPart);
            }
            for (String pvName : allNames) {
                if (pattern.matcher(pvName).matches()) {
                    ret.add(pvName);
                }
            }
            return ret;
        }
    }

    @Override
    public ApplianceAggregateInfo getAggregatedApplianceInfo(ApplianceInfo applianceInfo) throws IOException {
        if (applianceInfo.getIdentity().equals(myApplianceInfo.getIdentity()) && this.warFile == WAR_FILE.MGMT) {
            logger.debug("Returning local copy of appliance info for " + applianceInfo.getIdentity());
            return applianceAggregateInfo;
        } else {
            try {
                JSONObject aggregateInfo = GetUrlContent.getURLContentAsJSONObject(
                        applianceInfo.getMgmtURL() + "/getAggregatedApplianceInfo", false);
                JSONDecoder<ApplianceAggregateInfo> jsonDecoder = JSONDecoder
                        .getDecoder(ApplianceAggregateInfo.class);
                ApplianceAggregateInfo retval = new ApplianceAggregateInfo();
                jsonDecoder.decode(aggregateInfo, retval);
                return retval;
            } catch (Exception ex) {
                throw new IOException(ex);
            }
        }
    }

    @Override
    public void registerPVToAppliance(String pvName, ApplianceInfo applianceInfo)
            throws AlreadyRegisteredException {
        ApplianceInfo info = pv2appliancemapping.get(pvName);
        if (info != null)
            throw new AlreadyRegisteredException(info);
        pv2appliancemapping.put(pvName, applianceInfo);
    }

    @Override
    public PVTypeInfo getTypeInfoForPV(String pvName) {
        if (typeInfos.containsKey(pvName)) {
            logger.debug("Retrieving typeinfo from cache for pv " + pvName);
            return typeInfos.get(pvName);
        }

        return null;
    }

    @Override
    public void updateTypeInfoForPV(String pvName, PVTypeInfo typeInfo) {
        logger.debug("Updating typeinfo for " + pvName);
        if (!typeInfo.keyAlreadyGenerated()) {
            // This call should also typically set the chunk key in the type info.
            this.pvName2KeyConverter.convertPVNameToKey(pvName);
        }

        typeInfos.put(pvName, typeInfo);
    }

    @Override
    public void removePVFromCluster(String pvName) {
        logger.info("Removing PV from cluster.." + pvName);
        pv2appliancemapping.remove(pvName);
        pvsForThisAppliance.remove(pvName);
        typeInfos.remove(pvName);
        pausedPVsForThisAppliance.remove(pvName);
        String[] parts = this.pvName2KeyConverter.breakIntoParts(pvName);
        for (String part : parts) {
            ConcurrentSkipListSet<String> pvNamesForPart = parts2PVNamesForThisAppliance.get(part);
            if (pvNamesForPart != null) {
                pvNamesForPart.remove(pvName);
            }
        }
    }

    private class PVApplianceCombo implements Comparable<PVApplianceCombo> {
        String applianceIdentity;
        String pvName;

        public PVApplianceCombo(String applianceIdentity, String pvName) {
            this.applianceIdentity = applianceIdentity;
            this.pvName = pvName;
        }

        @Override
        public int compareTo(PVApplianceCombo other) {
            if (this.applianceIdentity.equals(other.applianceIdentity)) {
                return this.pvName.compareTo(other.pvName);
            } else {
                return this.applianceIdentity.compareTo(other.applianceIdentity);
            }
        }
    }

    private List<PVApplianceCombo> getSortedPVApplianceCombo() {
        ArrayList<PVApplianceCombo> sortedCombos = new ArrayList<PVApplianceCombo>();
        for (Map.Entry<String, ApplianceInfo> entry : pv2appliancemapping.entrySet()) {
            sortedCombos.add(new PVApplianceCombo(entry.getValue().getIdentity(), entry.getKey()));
        }
        Collections.sort(sortedCombos);
        return sortedCombos;
    }

    @Override
    public void addToArchiveRequests(String pvName, UserSpecifiedSamplingParams userSpecifiedSamplingParams) {
        archivePVRequests.put(pvName, userSpecifiedSamplingParams);
        try {
            persistanceLayer.putArchivePVRequest(pvName, userSpecifiedSamplingParams);
        } catch (IOException ex) {
            logger.error("Exception adding request to persistence", ex);
        }
    }

    @Override
    public void updateArchiveRequest(String pvName, UserSpecifiedSamplingParams userSpecifiedSamplingParams) {
        try {
            if (persistanceLayer.getArchivePVRequest(pvName) != null) {
                archivePVRequests.put(pvName, userSpecifiedSamplingParams);
                persistanceLayer.putArchivePVRequest(pvName, userSpecifiedSamplingParams);
            } else {
                logger.error(
                        "Do not have user specified params for pv " + pvName + " in this appliance. Not updating.");
            }
        } catch (IOException ex) {
            logger.error("Exception updating request in persistence", ex);
        }
    }

    @Override
    public Set<String> getArchiveRequestsCurrentlyInWorkflow() {
        return new HashSet<String>(archivePVRequests.keySet());
    }

    @Override
    public boolean doesPVHaveArchiveRequestInWorkflow(String pvname) {
        return archivePVRequests.containsKey(pvname);
    }

    @Override
    public UserSpecifiedSamplingParams getUserSpecifiedSamplingParams(String pvName) {
        return archivePVRequests.get(pvName);
    }

    @Override
    public void archiveRequestWorkflowCompleted(String pvName) {
        archivePVRequests.remove(pvName);
        try {
            persistanceLayer.removeArchivePVRequest(pvName);
        } catch (IOException ex) {
            logger.error("Exception removing request from persistence", ex);
        }
    }

    @Override
    public int getInitialDelayBeforeStartingArchiveRequestWorkflow() {
        int appliancesInCluster = 0;
        for (@SuppressWarnings("unused")
        ApplianceInfo info : this.getAppliancesInCluster()) {
            appliancesInCluster++;
        }

        int initialDelayInSeconds = 10;
        if (appliancesInCluster > 1) {
            // We use a longer initial delay here to get all the appliances in the cluster a chance to restart
            initialDelayInSeconds = 30 * 60;
        }

        return initialDelayInSeconds;
    }

    @Override
    public void addAlias(String aliasName, String realName) {
        aliasNamesToRealNames.put(aliasName, realName);
        try {
            persistanceLayer.putAliasNamesToRealName(aliasName, realName);
        } catch (IOException ex) {
            logger.error("Exception adding alias name to persistence " + aliasName, ex);
        }

        // Add aliases into the trie
        String[] parts = this.pvName2KeyConverter.breakIntoParts(aliasName);
        for (String part : parts) {
            if (!parts2PVNamesForThisAppliance.containsKey(part)) {
                parts2PVNamesForThisAppliance.put(part, new ConcurrentSkipListSet<String>());
            }
            parts2PVNamesForThisAppliance.get(part).add(aliasName);
        }
    }

    @Override
    public void removeAlias(String aliasName, String realName) {
        aliasNamesToRealNames.remove(aliasName);
        try {
            persistanceLayer.removeAliasName(aliasName, realName);
        } catch (IOException ex) {
            logger.error("Exception removing alias name from persistence " + aliasName, ex);
        }

        // Remove the aliasname from the trie
        String[] parts = this.pvName2KeyConverter.breakIntoParts(aliasName);
        for (String part : parts) {
            parts2PVNamesForThisAppliance.get(part).remove(aliasName);
        }
    }

    @Override
    public String getRealNameForAlias(String aliasName) {
        return aliasNamesToRealNames.get(aliasName);
    }

    @Override
    public List<String> getAllAliases() {
        return new ArrayList<String>(aliasNamesToRealNames.keySet());
    }

    private final String[] extraFields = new String[] { "MDEL", "ADEL", "SCAN", "RTYP" };

    @Override
    public String[] getExtraFields() {
        return extraFields;
    }

    @Override
    public Set<String> getRuntimeFields() {
        return runTimeFields;
    }

    @Override
    public PBThreeTierETLPVLookup getETLLookup() {
        return etlPVLookup;
    }

    @Override
    public RetrievalState getRetrievalRuntimeState() {
        return retrievalState;
    }

    @Override
    public boolean isShuttingDown() {
        return startupExecutor.isShutdown();
    }

    @Override
    public void addShutdownHook(Runnable runnable) {
        shutdownHooks.add(runnable);

    }

    private void runShutDownHooksAndCleanup() {
        LinkedList<Runnable> shutDnHooks = new LinkedList<Runnable>(this.shutdownHooks);
        Collections.reverse(shutDnHooks);
        logger.debug("Running shutdown hooks in webapp " + this.warFile);
        for (Runnable shutdownHook : shutDnHooks) {
            try {
                shutdownHook.run();
            } catch (Throwable t) {
                logger.warn("Exception shutting down service using shutdown hook " + shutdownHook.toString(), t);
            }
        }
        logger.debug("Done running shutdown hooks in webapp " + this.warFile);
    }

    @Override
    public void shutdownNow() {
        this.runShutDownHooksAndCleanup();
    }

    @Override
    public Map<String, String> getExternalArchiverDataServers() {
        return channelArchiverDataServers;
    }

    @Override
    public void addExternalArchiverDataServer(String serverURL, String archivesCSV) throws IOException {
        String[] archives = archivesCSV.split(",");
        boolean loadCAPVs = false;
        if (!this.getExternalArchiverDataServers().containsKey(serverURL)) {
            this.getExternalArchiverDataServers().put(serverURL, archivesCSV);
            loadCAPVs = true;
        } else {
            logger.info(
                    serverURL + " already exists in the map. So, skipping loading PVs from the external server.");
        }

        // We always add to persistence; whether this is from the UI or from the other appliances in the cluster.
        if (this.persistanceLayer != null) {
            persistanceLayer.putExternalDataServer(serverURL, archivesCSV);
        }

        try {
            // We load PVs from the external server only if this is the first server starting up...
            if (loadCAPVs) {
                for (int i = 0; i < archives.length; i++) {
                    String archive = archives[i];
                    loadExternalArchiverPVs(serverURL, archive);
                }
            }
        } catch (Exception ex) {
            logger.error("Exception adding Channel Archiver archives " + serverURL + " - " + archivesCSV, ex);
            throw new IOException(ex);
        }
    }

    @Override
    public void removeExternalArchiverDataServer(String serverURL, String archivesCSV) throws IOException {
        this.getExternalArchiverDataServers().remove(serverURL);
        // We always add to persistence; whether this is from the UI or from the other appliances in the cluster.
        if (this.persistanceLayer != null) {
            logger.info("Removing the channel archiver server " + serverURL + " from the persistent store.");
            persistanceLayer.removeExternalDataServer(serverURL, archivesCSV);
        }

    }

    /**
     * Given a external Archiver data server URL and an archive; 
     * If this is a ChannelArchiver (archives != pbraw); 
     * this adds the PVs in the Channel Archiver so that they can be proxied.
     * @param serverURL
     * @param archive
     * @throws IOException
     * @throws SAXException
     */
    private void loadExternalArchiverPVs(String serverURL, String archive) throws IOException, SAXException {
        if (archive.equals("pbraw")) {
            logger.debug(
                    "We do not load PV names from external EPICS archiver appliances. These can number in the multiple millions and the respone on retrieval is fast enough anyways");
            return;
        }

        ChannelArchiverDataServerInfo serverInfo = new ChannelArchiverDataServerInfo(serverURL, archive);
        NamesHandler handler = new NamesHandler();
        logger.debug(
                "Getting list of PV's from Channel Archiver Server at " + serverURL + " using index " + archive);
        XMLRPCClient.archiverNames(serverURL, archive, handler);
        HashMap<String, List<ChannelArchiverDataServerPVInfo>> tempPVNames = new HashMap<String, List<ChannelArchiverDataServerPVInfo>>();
        long totalPVsProxied = 0;
        for (NamesHandler.ChannelDescription pvChannelDesc : handler.getChannels()) {
            String pvName = PVNames.normalizePVName(pvChannelDesc.getName());
            if (this.pv2ChannelArchiverDataServer.containsKey(pvName)) {
                List<ChannelArchiverDataServerPVInfo> alreadyExistingServers = this.pv2ChannelArchiverDataServer
                        .get(pvName);
                logger.debug("Adding new server to already existing ChannelArchiver server for " + pvName);
                addExternalCAServerToExistingList(alreadyExistingServers, serverInfo, pvChannelDesc);
                tempPVNames.put(pvName, alreadyExistingServers);
            } else if (tempPVNames.containsKey(pvName)) {
                List<ChannelArchiverDataServerPVInfo> alreadyExistingServers = tempPVNames.get(pvName);
                logger.debug("Adding new server to already existing ChannelArchiver server (in tempspace) for "
                        + pvName);
                addExternalCAServerToExistingList(alreadyExistingServers, serverInfo, pvChannelDesc);
                tempPVNames.put(pvName, alreadyExistingServers);
            } else {
                List<ChannelArchiverDataServerPVInfo> caServersForPV = new ArrayList<ChannelArchiverDataServerPVInfo>();
                caServersForPV.add(new ChannelArchiverDataServerPVInfo(serverInfo, pvChannelDesc.getStartSec(),
                        pvChannelDesc.getEndSec()));
                tempPVNames.put(pvName, caServersForPV);
            }

            if (tempPVNames.size() > 1000) {
                this.pv2ChannelArchiverDataServer.putAll(tempPVNames);
                totalPVsProxied += tempPVNames.size();
                tempPVNames.clear();
            }
        }
        if (!tempPVNames.isEmpty()) {
            this.pv2ChannelArchiverDataServer.putAll(tempPVNames);
            totalPVsProxied += tempPVNames.size();
            tempPVNames.clear();
        }
        if (logger.isDebugEnabled())
            logger.debug("Proxied a total of " + totalPVsProxied + " from server " + serverURL + " using archive "
                    + archive);

    }

    private static void addExternalCAServerToExistingList(
            List<ChannelArchiverDataServerPVInfo> alreadyExistingServers, ChannelArchiverDataServerInfo serverInfo,
            NamesHandler.ChannelDescription pvChannelDesc) {
        List<ChannelArchiverDataServerPVInfo> copyOfAlreadyExistingServers = new LinkedList<ChannelArchiverDataServerPVInfo>();
        for (ChannelArchiverDataServerPVInfo alreadyExistingServer : alreadyExistingServers) {
            if (alreadyExistingServer.getServerInfo().equals(serverInfo)) {
                logger.debug("Removing a channel archiver server that already exists "
                        + alreadyExistingServer.toString());
            } else {
                copyOfAlreadyExistingServers.add(alreadyExistingServer);
            }
        }

        int beforeCount = alreadyExistingServers.size();
        alreadyExistingServers.clear();
        alreadyExistingServers.addAll(copyOfAlreadyExistingServers);

        // Readd the CA server - this should take into account any updated start times, end times and so on.
        alreadyExistingServers.add(new ChannelArchiverDataServerPVInfo(serverInfo, pvChannelDesc.getStartSec(),
                pvChannelDesc.getEndSec()));

        int afterCount = alreadyExistingServers.size();
        logger.debug("We had " + beforeCount + " and now we have " + afterCount
                + " when adding external ChannelArchiver server");

        // Sort the servers by ascending time stamps before adding it back.
        ChannelArchiverDataServerPVInfo.sortServersBasedOnStartAndEndSecs(alreadyExistingServers);
    }

    @Override
    public List<ChannelArchiverDataServerPVInfo> getChannelArchiverDataServers(String pvName) {
        String normalizedPVName = PVNames.normalizePVName(pvName);
        logger.debug("Looking for CA sever for pv " + normalizedPVName);
        return pv2ChannelArchiverDataServer.get(normalizedPVName);
    }

    @Override
    public PolicyConfig computePolicyForPV(String pvName, MetaInfo metaInfo,
            UserSpecifiedSamplingParams userSpecParams) throws IOException {
        try (InputStream is = this.getPolicyText()) {
            logger.debug("Computing policy for pvName");
            HashMap<String, Object> pvInfo = new HashMap<String, Object>();
            pvInfo.put("dbrtype", metaInfo.getArchDBRTypes().toString());
            pvInfo.put("elementCount", metaInfo.getCount());
            pvInfo.put("eventRate", metaInfo.getEventRate());
            pvInfo.put("storageRate", metaInfo.getStorageRate());
            pvInfo.put("aliasName", metaInfo.getAliasName());
            if (userSpecParams != null && userSpecParams.getPolicyName() != null) {
                logger.debug("Passing user override of policy " + userSpecParams.getPolicyName()
                        + " as the dict entry policyName");
                pvInfo.put("policyName", userSpecParams.getPolicyName());
            }
            if (userSpecParams.getControllingPV() != null) {
                pvInfo.put("controlPV", userSpecParams.getControllingPV());
            }

            HashMap<String, String> otherMetaInfo = metaInfo.getOtherMetaInfo();
            for (String otherMetaInfoKey : this.getExtraFields()) {
                if (otherMetaInfo.containsKey(otherMetaInfoKey)) {
                    if (otherMetaInfoKey.equals("ADEL") || otherMetaInfoKey.equals("MDEL")) {
                        try {
                            pvInfo.put(otherMetaInfoKey, Double.parseDouble(otherMetaInfo.get(otherMetaInfoKey)));
                        } catch (Exception ex) {
                            logger.error("Exception adding MDEL and ADEL to the info", ex);
                        }
                    } else {
                        pvInfo.put(otherMetaInfoKey, otherMetaInfo.get(otherMetaInfoKey));
                    }
                }
            }

            if (logger.isDebugEnabled()) {
                StringBuilder buf = new StringBuilder();
                buf.append("Before computing policy for");
                buf.append(pvName);
                buf.append(" pvInfo is \n");
                for (String key : pvInfo.keySet()) {
                    buf.append(key);
                    buf.append("=");
                    buf.append(pvInfo.get(key));
                    buf.append("\n");
                }
                logger.debug(buf.toString());
            }

            try {
                // We only have one policy in the cache...
                ExecutePolicy executePolicy = theExecutionPolicy.get("ThePolicy");
                PolicyConfig policyConfig = executePolicy.computePolicyForPV(pvName, pvInfo);
                return policyConfig;
            } catch (ExecutionException e) {
                Throwable cause = e.getCause();
                logger.error("Exception executing policy for pv " + pvName, cause);
                if (cause instanceof IOException) {
                    throw (IOException) cause;
                } else {
                    throw new IOException(cause);
                }
            }
        }
    }

    @Override
    public HashMap<String, String> getPoliciesInInstallation() throws IOException {
        try (ExecutePolicy executePolicy = new ExecutePolicy(this)) {
            return executePolicy.getPolicyList();
        }
    }

    @Override
    public List<String> getFieldsArchivedAsPartOfStream() throws IOException {
        try (ExecutePolicy executePolicy = new ExecutePolicy(this)) {
            return executePolicy.getFieldsArchivedAsPartOfStream();
        }
    }

    @Override
    public TypeSystem getArchiverTypeSystem() {
        return new PBTypeSystem();
    }

    private boolean finishedLoggingPolicyLocation = false;

    @Override
    public InputStream getPolicyText() throws IOException {
        String policiesPyFile = System.getProperty(ARCHAPPL_POLICIES);
        if (policiesPyFile == null) {
            policiesPyFile = System.getenv(ARCHAPPL_POLICIES);
            if (policiesPyFile != null) {
                if (!finishedLoggingPolicyLocation) {
                    configlogger.info("Obtained policies location from environment " + policiesPyFile);
                    finishedLoggingPolicyLocation = true;
                }
                return new FileInputStream(new File(policiesPyFile));
            } else {
                logger.info("Looking for /WEB-INF/classes/policies.py in classpath");
                if (servletContext != null) {
                    if (!finishedLoggingPolicyLocation) {
                        configlogger.info("Using policies file /WEB-INF/classes/policies.py found in classpath");
                        finishedLoggingPolicyLocation = true;
                    }
                    return servletContext.getResourceAsStream("/WEB-INF/classes/policies.py");
                } else {
                    throw new IOException(
                            "Cannot determine location of policies file as both servlet context and webInfClassesFolder are null");
                }
            }
        } else {
            if (!finishedLoggingPolicyLocation) {
                configlogger.info("Obtained policies location from system property " + policiesPyFile);
                finishedLoggingPolicyLocation = true;
            }
            return new FileInputStream(new File(policiesPyFile));
        }
    }

    @Override
    public EngineContext getEngineContext() {
        return engineContext;
    }

    @Override
    public MgmtRuntimeState getMgmtRuntimeState() {
        return mgmtRuntime;
    }

    @Override
    public WAR_FILE getWarFile() {
        return warFile;
    }

    /**
     * Return a string representation of the member.
     * @param member
     * @return
     */
    private String getMemberKey(Member member) {
        // We use deprecated versions of the methods as the non-deprecated versions do not work as of 2.0x?
        return member.getSocketAddress().toString();
    }

    @Override
    public EventBus getEventBus() {
        return eventBus;
    }

    @Override
    public PVNameToKeyMapping getPVNameToKeyConverter() {
        return pvName2KeyConverter;
    }

    /**
     * Load typeInfos into the cluster hashmaps from the persistence layer on startup.
     * To avoid overwhelming the cluster, we batch the loads 
     */
    private void loadTypeInfosFromPersistence() {
        try {
            configlogger.info("Loading PVTypeInfo from persistence");
            List<String> upgradedPVs = new LinkedList<String>();
            List<String> pvNamesFromPersistence = persistanceLayer.getTypeInfoKeys();
            HashMap<String, PVTypeInfo> newTypeInfos = new HashMap<String, PVTypeInfo>();
            HashMap<String, ApplianceInfo> newPVMappings = new HashMap<String, ApplianceInfo>();
            int objectCount = 0;
            int batch = 0;
            int clusterPVCount = 0;
            for (String pvNameFromPersistence : pvNamesFromPersistence) {
                PVTypeInfo typeInfo = persistanceLayer.getTypeInfo(pvNameFromPersistence);
                if (typeInfo.getApplianceIdentity().equals(myIdentity)) {
                    // Here's where we put schema update logic
                    upgradeTypeInfo(typeInfo, upgradedPVs);

                    String pvName = typeInfo.getPvName();
                    newTypeInfos.put(pvName, typeInfo);
                    newPVMappings.put(pvName, appliances.get(typeInfo.getApplianceIdentity()));
                    pvsForThisAppliance.add(pvName);
                    if (typeInfo.isPaused()) {
                        pausedPVsForThisAppliance.add(pvName);
                    }
                    String[] parts = this.pvName2KeyConverter.breakIntoParts(pvName);
                    for (String part : parts) {
                        if (!parts2PVNamesForThisAppliance.containsKey(part)) {
                            parts2PVNamesForThisAppliance.put(part, new ConcurrentSkipListSet<String>());
                        }
                        parts2PVNamesForThisAppliance.get(part).add(pvName);
                    }

                    objectCount++;
                }
                // Add in batch sizes of 1000 or so...
                if (objectCount > 1000) {
                    this.typeInfos.putAll(newTypeInfos);
                    this.pv2appliancemapping.putAll(newPVMappings);
                    for (String pvName : newTypeInfos.keySet()) {
                        applianceAggregateInfo.addInfoForPV(pvName, newTypeInfos.get(pvName), this);
                    }
                    clusterPVCount += newTypeInfos.size();
                    newTypeInfos = new HashMap<String, PVTypeInfo>();
                    newPVMappings = new HashMap<String, ApplianceInfo>();
                    objectCount = 0;
                    logger.debug("Adding next batch of PVs " + batch++);
                }
            }

            if (newTypeInfos.size() > 0) {
                logger.debug("Adding final batch of PVs from persistence");
                this.typeInfos.putAll(newTypeInfos);
                this.pv2appliancemapping.putAll(newPVMappings);
                for (String pvName : newTypeInfos.keySet()) {
                    applianceAggregateInfo.addInfoForPV(pvName, newTypeInfos.get(pvName), this);
                }
                clusterPVCount += newTypeInfos.size();
            }

            configlogger.info("Done loading " + +clusterPVCount + " PVs from persistence into cluster");

            for (String upgradedPVName : upgradedPVs) {
                logger.debug("PV " + upgradedPVName + "'s schema was upgraded");
                persistanceLayer.putTypeInfo(upgradedPVName, getTypeInfoForPV(upgradedPVName));
                logger.debug("Done persisting upgraded PV's " + upgradedPVName + "'s typeInfo");
            }
        } catch (Exception ex) {
            configlogger.error("Exception loading PVs from persistence", ex);
        }
    }

    /**
     * The occasional upgrade to PVTypeInfo schema is handed here.
     * @param typeInfo - Typeinfo to be upgraded
     * @param upgradedPVs - Add the pvName here if we actually did an upgrade to the typeInfo.
     */
    private void upgradeTypeInfo(PVTypeInfo typeInfo, List<String> upgradedPVs) {
        // We added the chunkKey to typeInfo to permanently remember the key mapping to accomodate slowly changing key mappings. 
        // This could be a possibility after talking to SPEAR folks.
        if (!typeInfo.keyAlreadyGenerated()) {
            typeInfo.setChunkKey(this.pvName2KeyConverter.convertPVNameToKey(typeInfo.getPvName()));
            upgradedPVs.add(typeInfo.getPvName());
        }
    }

    /**
     * Load alias mappings from persistence on startup in batches
     */
    private void loadAliasesFromPersistence() {
        try {
            configlogger.info("Loading aliases from persistence");
            List<String> pvNamesFromPersistence = persistanceLayer.getAliasNamesToRealNamesKeys();
            HashMap<String, String> newAliases = new HashMap<String, String>();
            int objectCount = 0;
            int batch = 0;
            int clusterPVCount = 0;
            for (String pvNameFromPersistence : pvNamesFromPersistence) {
                String realName = persistanceLayer.getAliasNamesToRealName(pvNameFromPersistence);
                if (this.pvsForThisAppliance.contains(realName)) {
                    newAliases.put(pvNameFromPersistence, realName);
                    // Add the alias into the trie
                    String[] parts = this.pvName2KeyConverter.breakIntoParts(pvNameFromPersistence);
                    for (String part : parts) {
                        if (!parts2PVNamesForThisAppliance.containsKey(part)) {
                            parts2PVNamesForThisAppliance.put(part, new ConcurrentSkipListSet<String>());
                        }
                        parts2PVNamesForThisAppliance.get(part).add(pvNameFromPersistence);
                    }
                }
                // Add in batch sizes of 1000 or so...
                if (objectCount > 1000) {
                    this.aliasNamesToRealNames.putAll(newAliases);
                    clusterPVCount += newAliases.size();
                    newAliases = new HashMap<String, String>();
                    objectCount = 0;
                    logger.debug("Adding next batch of aliases " + batch++);
                }
            }

            if (newAliases.size() > 0) {
                logger.debug("Adding final batch of aliases from persistence");
                this.aliasNamesToRealNames.putAll(newAliases);
                clusterPVCount += newAliases.size();
            }

            configlogger.info("Done loading " + clusterPVCount + " aliases from persistence into cluster ");
        } catch (Exception ex) {
            configlogger.error("Exception loading aliases from persistence", ex);
        }
    }

    /**
     * Load any pending archive requests that have not been fulfilled yet on startup
     * Also, start their workflows..
     * @throws ConfigException
     */
    private void loadArchiveRequestsFromPersistence() throws ConfigException {
        try {
            configlogger.info("Loading archive requests from persistence");
            List<String> pvNamesFromPersistence = persistanceLayer.getArchivePVRequestsKeys();
            HashMap<String, UserSpecifiedSamplingParams> newArchiveRequests = new HashMap<String, UserSpecifiedSamplingParams>();
            int objectCount = 0;
            int batch = 0;
            int clusterPVCount = 0;
            for (String pvNameFromPersistence : pvNamesFromPersistence) {
                UserSpecifiedSamplingParams userSpecifiedParams = persistanceLayer
                        .getArchivePVRequest(pvNameFromPersistence);
                // We should not need to add an appliance check here.. However, if after production deployment, we determine we need to do so; this is the right place.
                newArchiveRequests.put(pvNameFromPersistence, userSpecifiedParams);
                // Add in batch sizes of 1000 or so...
                if (objectCount > 1000) {
                    this.archivePVRequests.putAll(newArchiveRequests);
                    clusterPVCount += newArchiveRequests.size();
                    newArchiveRequests = new HashMap<String, UserSpecifiedSamplingParams>();
                    objectCount = 0;
                    logger.debug("Adding next batch of archive pv requests " + batch++);
                }
            }

            if (newArchiveRequests.size() > 0) {
                logger.debug("Adding final batch of archive pv requests from persistence");
                this.archivePVRequests.putAll(newArchiveRequests);
                clusterPVCount += newArchiveRequests.size();
            }

            configlogger
                    .info("Done loading " + clusterPVCount + " archive pv requests from persistence into cluster ");
        } catch (Exception ex) {
            configlogger.error("Exception loading archive pv requests from persistence", ex);
        }
    }

    /**
     * Initialize the persistenceLayer using environment/system property.
     * By default, initialize the MySQLPersistence
     * @throws ConfigException
     */
    private void initializePersistenceLayer() throws ConfigException {
        String persistenceFromEnv = System.getenv(ARCHAPPL_PERSISTENCE_LAYER);
        if (persistenceFromEnv == null || persistenceFromEnv.equals("")) {
            persistenceFromEnv = System.getProperty(ARCHAPPL_PERSISTENCE_LAYER);
        }
        if (persistenceFromEnv == null || persistenceFromEnv.equals("")) {
            logger.info(
                    "Using MYSQL for persistence; we expect to find a JNDI connection pool called jdbc/archappl");
            persistanceLayer = new MySQLPersistence();
        } else {
            try {
                logger.info("Using persistence provided by class " + persistenceFromEnv);
                persistanceLayer = (ConfigPersistence) getClass().getClassLoader().loadClass(persistenceFromEnv)
                        .newInstance();
            } catch (Exception ex) {
                throw new ConfigException("Exception initializing persistence layer using " + persistenceFromEnv,
                        ex);
            }
        }
    }

    public ProcessMetrics getProcessMetrics() {
        return processMetrics;
    }

    @Override
    public String getWebInfFolder() {
        return servletContext.getRealPath("WEB-INF/");
    }

    private void loadExternalServersFromPersistence() throws ConfigException {
        try {
            configlogger.info("Loading external servers from persistence");
            List<String> externalServerKeys = persistanceLayer.getExternalDataServersKeys();
            for (String serverUrl : externalServerKeys) {
                String archivesCSV = persistanceLayer.getExternalDataServer(serverUrl);
                if (this.getExternalArchiverDataServers().containsKey(serverUrl)) {
                    configlogger.info("Skipping adding " + serverUrl
                            + " on this appliance as another appliance has already added it");
                } else {
                    this.getExternalArchiverDataServers().put(serverUrl, archivesCSV);
                    String[] archives = archivesCSV.split(",");

                    this.startupExecutor.schedule(new Runnable() {
                        @Override
                        public void run() {
                            try {
                                for (int i = 0; i < archives.length; i++) {
                                    String archive = archives[i];
                                    loadExternalArchiverPVs(serverUrl, archive);
                                }
                            } catch (Exception ex) {
                                logger.error("Exception adding Channel Archiver archives " + serverUrl + " - "
                                        + archivesCSV, ex);
                            }
                        }
                    }, 15, TimeUnit.SECONDS);
                }
            }
            configlogger.info("Done loading external servers from persistence ");
        } catch (Exception ex) {
            configlogger.error("Exception loading external servers from persistence", ex);
        }
    }

    private void registerForNewExternalServers(IMap<Object, Object> dataServerMap) throws ConfigException {
        dataServerMap.addEntryListener(new EntryAddedListener<Object, Object>() {
            @Override
            public void entryAdded(EntryEvent<Object, Object> arg0) {
                String url = (String) arg0.getKey();
                String archivesCSV = (String) arg0.getValue();
                try {
                    addExternalArchiverDataServer(url, archivesCSV);
                } catch (Exception ex) {
                    logger.error("Exception syncing external data server " + url + archivesCSV, ex);
                }
            }
        }, true);
        dataServerMap.addEntryListener(new EntryRemovedListener<Object, Object>() {
            @Override
            public void entryRemoved(EntryEvent<Object, Object> arg0) {
                String url = (String) arg0.getKey();
                String archivesCSV = (String) arg0.getValue();
                try {
                    removeExternalArchiverDataServer(url, archivesCSV);
                } catch (Exception ex) {
                    logger.error("Exception syncing external data server " + url + archivesCSV, ex);
                }
            }
        }, true);
    }

    @Override
    public Set<String> getPausedPVsInThisAppliance() {
        if (pausedPVsForThisAppliance != null) {
            logger.debug("Returning the locally cached copy of the paused pvs for this appliance");
            return pausedPVsForThisAppliance;
        } else {
            logger.debug("Fetching the list of paused PVs for this appliance from the mgmt app");
            JSONArray pvs = GetUrlContent
                    .getURLContentAsJSONArray(myApplianceInfo.getMgmtURL() + "/getPausedPVsForThisAppliance");
            HashSet<String> retval = new HashSet<String>();
            for (Object pv : pvs) {
                retval.add((String) pv);
            }
            return retval;
        }
    }

    @Override
    public void refreshPVDataFromChannelArchiverDataServers() {
        Map<String, String> existingCAServers = this.getExternalArchiverDataServers();
        for (String serverURL : existingCAServers.keySet()) {
            String archivesCSV = existingCAServers.get(serverURL);
            String[] archives = archivesCSV.split(",");

            try {
                for (int i = 0; i < archives.length; i++) {
                    String archive = archives[i];
                    loadExternalArchiverPVs(serverURL, archive);
                }
            } catch (Throwable ex) {
                logger.error("Exception adding Channel Archiver archives " + serverURL + " - " + archivesCSV, ex);
            }
        }
    }

    @Override
    public boolean getNamedFlag(String name) {
        if (name == null) {
            return false;
        }
        if (name.equalsIgnoreCase("false")) {
            return false;
        }
        if (name.equalsIgnoreCase("true")) {
            return true;
        }
        if (namedFlags.containsKey(name)) {
            return namedFlags.get(name);
        }
        // If we don't know about this named flag, we return false;
        return false;
    }

    @Override
    public void setNamedFlag(String name, boolean value) {
        namedFlags.put(name, value);
    }

    @Override
    public Set<String> getNamedFlagNames() {
        return namedFlags.keySet();
    }
}