com.pari.nm.modules.jobs.VSEMImporter.java Source code

Java tutorial

Introduction

Here is the source code for com.pari.nm.modules.jobs.VSEMImporter.java

Source

package com.pari.nm.modules.jobs;

import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.zip.ZipInputStream;

import org.apache.commons.io.FileUtils;
import org.apache.commons.jelly.JellyTagException;
import org.apache.log4j.BasicConfigurator;
import org.w3c.dom.Document;
import org.w3c.dom.Element;

import com.cisco.vse.entities.definition.EntityDefLoader;
import com.cisco.vse.entities.definition.EntityDefRepository;
import com.cisco.vse.entities.runtime.EntityData;
import com.cisco.vse.entities.runtime.EntityDataProviderIf;
import com.cisco.vse.entities.runtime.EntityVector;
import com.cisco.vse.persistence.Format;
import com.cisco.vse.persistence.VSEFileProcessor;
import com.pari.api.PariAPIException;
import com.pari.api.PariAPIMessages;
import com.pari.api.utils.XMLUtil;
import com.pari.base.so.ModuleInfo;
import com.pari.base.so.ModuleType;
import com.pari.base.so.NetworkNode;
import com.pari.ic.ICManager;
import com.pari.ic.ICStatisticsManager;
import com.pari.jelly.engine.communication.ISnmpScalar;
import com.pari.jms.events.PerDeviceConfigBackupStatus;
import com.pari.jms.events.PerDeviceImportStatus;
import com.pari.logger.PariLogger;
import com.pari.logger.PariLoggerFactory;
import com.pari.nm.gui.guimessages.ClientMessageType;
import com.pari.nm.gui.guimessages.DeviceStateChangeMessage;
import com.pari.nm.modules.customers.CustomerIfImpl;
import com.pari.nm.modules.customers.CustomerInstance;
import com.pari.nm.modules.dsp.ConfigResyncDSPHandler;
import com.pari.nm.modules.dsp.DSPHandler;
import com.pari.nm.modules.dsp.DSPICExecutor;
import com.pari.nm.modules.dsp.DSPMetadataHandler;
import com.pari.nm.modules.dsp.DSPScriptCache;
import com.pari.nm.modules.dsp.InventoryDSPHandler;
import com.pari.nm.modules.dsp.management.DeviceScriptInfo;
import com.pari.nm.modules.dsp.management.DeviceScriptInfo.PackageType;
import com.pari.nm.modules.dsp.management.DeviceScriptManagerImpl;
import com.pari.nm.modules.dsp.pojo.DSPConfigData;
import com.pari.nm.modules.dsp.pojo.DSPNewDevice;
import com.pari.nm.modules.inventory.DeviceModule;
import com.pari.nm.modules.messaging.Messenger;
import com.pari.nm.modules.msgdefs.JobStatusIf;
import com.pari.nm.modules.msgdefs.MessageTypes;
import com.pari.nm.modules.msgdefs.VirtualDeviceRefresh;
import com.pari.nm.modules.session.ClientSessionManager;
import com.pari.nm.modules.session.DeviceAdder;
import com.pari.nm.modules.session.DeviceAdder.InsertMode;
import com.pari.nm.modules.session.NetworkNodeCache;
import com.pari.nm.modules.session.NmIfImpl;
import com.pari.nm.modules.topology.DeviceManager;
import com.pari.nm.utils.Constants;
import com.pari.nm.utils.NCCMThreadPoolService;
import com.pari.nm.utils.backup.UnzipDir;
import com.pari.nm.utils.db.InventoryDBHelper;
import com.pari.nm.utils.db.ServerDBHelper;
import com.pari.nm.utils.snmp.SNMPProductModelObject;
import com.pari.nm.utils.snmp.SNMPProductModelRepository;
import com.pari.pcb.VSEMCollectionInfo;
import com.pari.pcb.VSEMCollectionLog;
import com.pari.product.ServerConstants;
import com.pari.swim.ScriptInfo;
import com.pari.swim.ScriptInfo.ScriptDefinition;
import com.pari.swim.archive.BackupSoftwareImageRequestProcessor;

/**
 * VSEMImporter job responsible for importing VSEM file in to NCCM database.
 */
public class VSEMImporter {
    private static PariLogger logger = PariLoggerFactory.getLogger(Constants.NM_LOGGER);
    String vseFileName = null;
    private JobStatusIf invoker;
    private String login;
    private int customerId = -1;
    private String wingInstanceName = null;
    private int rowId = -1;
    public static final SimpleDateFormat format = new SimpleDateFormat("yyyy MMM dd HH:mm:ss");
    private int numDevs = 0;
    DSPICExecutor dspExecutor = new DSPICExecutor();
    DSPHandler<DSPNewDevice> dspHandler = new InventoryDSPHandler();

    private static ExecutorService executor = NCCMThreadPoolService.newShrinkingThreadPool("VSEMImporter.executor",
            ServerConstants.getInstance().get(ServerConstants.Types.VsemImportWorkerThreads), 5 * 60);
    private AtomicInteger numDevsDone = new AtomicInteger();
    private VSEMCollectionInfo collInfo = null;
    private String grpName;
    Map<String/* deivceIp */, PerDeviceImportStatus> perDeviceImportStatus = new ConcurrentHashMap<String, PerDeviceImportStatus>();
    Map<String/* deivceIp */, List<DAVStatus>> davStatusData = new ConcurrentHashMap<String, List<DAVStatus>>();
    Map<String/* deivceIp */, PerDeviceConfigBackupStatus> perDeviceConfigBackupStatus = null;
    private Map<String, Integer> nxosDevices = new ConcurrentHashMap<String, Integer>();
    private Map<String, Integer> wlcDevices = null;
    private Map<String, Integer> ucsDevices = new ConcurrentHashMap<String, Integer>();
    private Map<String, String> onsDevices = new ConcurrentHashMap<String, String>();
    private Map<String, Integer> junosDevices = new ConcurrentHashMap<String, Integer>();
    private Map<String, Map<String, String>> otherDevices = new ConcurrentHashMap<String, Map<String, String>>();
    private int jobId = -1;

    private static int subArraySize = 40;
    private JobParameters jobParameters = null;

    public VSEMImporter(String vseFileName, int customerId, String wingInstanceName, String grpName,
            JobStatusIf invoker, int rowId, String login, int jobId, JobParameters jobParameters) {
        this.vseFileName = vseFileName;
        this.rowId = rowId;
        this.customerId = customerId;
        this.wingInstanceName = wingInstanceName;
        this.invoker = invoker;
        this.login = login;
        this.grpName = grpName;
        this.jobId = jobId;
        this.jobParameters = jobParameters;
    }

    /**
     * Import vsem.
     *
     * @throws Exception
     *             the exception
     */
    public void importVsem() throws Exception {
        long start = System.currentTimeMillis();
        // Importing from VSEM file
        // it is not scaling well if we are processing VSEM files directly from zip.
        // extract it to a temp directory and process it.
        File rootDir = null;
        File pcb = new File(vseFileName);
        if (pcb.isDirectory()) {
            rootDir = pcb;
        } else {
            File tempDir = new File(pcb.getParent(), "TEMP");
            String f = pcb.getName();
            if (f.length() > 20) {
                f = f.substring(0, 20);
            }
            f += System.nanoTime();
            rootDir = new File(tempDir, f);
            rootDir.mkdirs();
        }
        try {
            BackupSoftwareImageRequestProcessor.getInstance().LockSchedule();
            VSEFileProcessor fp = getVSE(pcb, vseFileName, rootDir);
            String fileName = null;
            if (org.apache.commons.lang.StringUtils.ordinalIndexOf(vseFileName, "_", 2) != -1) {
                fileName = vseFileName.substring(0,
                        org.apache.commons.lang.StringUtils.ordinalIndexOf(vseFileName, "_", 2));
            }
            importFromFileProcessor(fp, fileName, rootDir);

            /*
             * Creating checkpoint files for NXOS devices after Vsem import is complete. If Vsem is collected as part of
             * CP executed before executing JOB(ApplyConfig/Rollback), then we don't run this thread.
             */

            if (collInfo == null || collInfo.getDescription() == null
                    || !collInfo.getDescription().contains("_NCCM_CP_BEFORE_")) {
                executor.execute(new NXOSCheckPointThread(nxosDevices));
                executor.execute(new WLCConfigCollectThread(wlcDevices));
                executor.execute(new UCSConfigCollectThread(ucsDevices));

                executor.execute(new ONSConfigDBThread(onsDevices));
                executor.execute(new JUNOSConfigCollectorThread(junosDevices));
                executor.execute(new OtherOSConfigCollectorThread(otherDevices));
            }
        } finally {
            try {
                new Thread(() -> {
                    {
                        try {
                            Thread.currentThread().sleep(30 * 60 * 1000);
                        } catch (Exception e) {
                            logger.warn("Error while the thread is in sleep.", e);
                        } finally {
                            BackupSoftwareImageRequestProcessor.getInstance().releaseScheduleLock();
                        }

                    }
                }, "BackupSoftwareImageRequestProcessorThread").start();
                if (rootDir != null) {
                    FileUtils.deleteDirectory(rootDir);
                }
            } catch (Exception ee) {
                logger.error("Exception while deleting temporary root directory", ee);
            }
        }

        logger.debug("Import VSEM job Execution Time-->" + (System.currentTimeMillis() - start));

    }

    private VSEMCollectionInfo readCollectionInfo(EntityData collInfo, EntityVector logList) {
        try {
            VSEMCollectionInfo colInfo = new VSEMCollectionInfo();
            colInfo.setId((String) collInfo.getPropValue("Id"));
            colInfo.setExecutionId((String) collInfo.getPropValue("ExecutionId"));
            colInfo.setTitle((String) collInfo.getPropValue("Title"));
            colInfo.setDescription((String) collInfo.getPropValue("Description"));
            String cpDesc = collInfo.getPropValue("Description") == null ? ""
                    : (String) collInfo.getPropValue("Description");
            onsDevices.put("CPDescription", cpDesc);
            Object timeObj = collInfo.getPropValue("StartTime");
            Long sTime = new Long(0);
            Long eTime = new Long(0);
            if (timeObj != null) {
                try {
                    sTime = new Long((String) timeObj);
                } catch (Exception e) {
                }
            }
            colInfo.setStartTime(sTime);

            timeObj = collInfo.getPropValue("EndTime");
            if (timeObj != null) {
                try {
                    eTime = new Long((String) timeObj);
                } catch (Exception ex) {
                }
            }
            colInfo.setEndTime(eTime);
            colInfo.setStatus((String) collInfo.getPropValue("Status"));
            if (logList != null && !logList.isEmpty()) {
                EntityData[] logs = logList.getEntities();
                colInfo.setLogList(new ArrayList<VSEMCollectionLog>());
                for (int idx = 0; idx < logs.length; idx++) {
                    try {
                        VSEMCollectionLog l = new VSEMCollectionLog();
                        l.setLogTime(Long.valueOf((String) logs[idx].getPropValue("Time")));
                        l.setMsg((String) logs[idx].getPropValue("Msg"));
                        colInfo.addLog(l);
                    } catch (Exception ex) {
                    }
                }
            }
            return colInfo;
        } catch (Exception e) {
            logger.warn("Error while setting collection info in setCollectionInfo", e);
            return null;
        }
    }

    /**
     * This method retrieves device DAV results from the zip CPExport_XXXXXXXX.
     *
     * @param rootDir
     */
    private Map<String, List<DAVStatus>> parseDAVData(String fileName, File rootDir) {
        try {
            if (fileName != null) {
                String file = rootDir.getAbsolutePath() + File.separator + fileName + ".xml";
                StringBuilder content = new StringBuilder();
                BufferedReader bufferedReader = new BufferedReader(new FileReader(file));

                String sCurrentLine;

                while ((sCurrentLine = bufferedReader.readLine()) != null) {
                    content.append(sCurrentLine);
                }
                bufferedReader.close();
                Document document = XMLUtil
                        .parseXMLDocument(new ByteArrayInputStream(content.toString().getBytes()));
                Element docElement = document.getDocumentElement();
                if (docElement != null) {

                    List<Element> davStatusElementList = XMLUtil.getFirstLevelChildElementsByTagName(docElement,
                            "DAVStatus");
                    DAVStatus davstatus = null;

                    if (davStatusElementList != null) {
                        for (Element davStatusElement : davStatusElementList) {
                            Element deviceElement = XMLUtil.getFirstLevelChildElementByTagName(davStatusElement,
                                    "Device");
                            String deviceIp = XMLUtil.getChildText(deviceElement, "IPAddress");
                            String protocol = XMLUtil.getChildText(davStatusElement, "Protocol");
                            String status = XMLUtil.getChildText(davStatusElement, "Status");

                            davstatus = new DAVStatus();
                            davstatus.setIpAddress(deviceIp);
                            davstatus.setProtocol(protocol);
                            davstatus.setStatus(status);
                            if (davStatusData.get(deviceIp) != null) {

                                davStatusData.get(deviceIp).add(davstatus);
                            } else {
                                List<DAVStatus> davStatusList = new ArrayList<DAVStatus>();
                                davStatusList.add(davstatus);
                                davStatusData.put(deviceIp, davStatusList);
                            }
                        }
                    }
                }
            }
        } catch (Exception exception) {
            logger.error("Failed parsing DAV Data.", exception);
        }
        return davStatusData;
    }

    private void importFromFileProcessor(VSEFileProcessor fp, String fileName, File rootDir)
            throws PariAPIException, MalformedURLException, JellyTagException, UnsupportedEncodingException,
            Exception {
        parseDAVData(fileName, rootDir);
        EntityDataProviderIf repo = fp.getDataProvider();
        EntityData network = repo.getEntity("Network");
        EntityVector devicesList = (EntityVector) network.getPropValue("DeviceList");
        EntityData[] devices = devicesList.getEntities();
        numDevs = (devices == null) ? 0 : devices.length;

        try {
            String identifier = new String();
            List<ScriptInfo> icfScripts = DeviceScriptManagerImpl.getInstance()
                    .getIcfDefinedZipPackages(PackageType.ZipInventory, customerId, "Icf");

            if (icfScripts != null) {
                Iterator<ScriptInfo> itr = icfScripts.iterator();
                while (itr.hasNext()) {
                    ScriptInfo script = itr.next();
                    identifier = script.getIdentifier();
                    break;
                }
                ICManager.getInstance().validateLicense(customerId, identifier, numDevs);
            }

            List<ScriptInfo> icfInvScripts = DeviceScriptManagerImpl.getInstance()
                    .getIcfDefinedZipPackages(PackageType.Inventory, customerId, "Icf");

            if (icfInvScripts != null) {
                Iterator<ScriptInfo> itr = icfInvScripts.iterator();
                while (itr.hasNext()) {
                    ScriptInfo script = itr.next();
                    identifier = script.getIdentifier();
                    break;
                }
                ICManager.getInstance().validateLicense(customerId, identifier, numDevs);
            }
        } catch (Exception e) {
            logger.warn("No sufficient license to run " + " - " + e.getMessage());
            logger.warn("No sufficient license to run import devices.");
            throw new Exception(e.getMessage());
        }
        if (network == null) {
            throw PariAPIMessages._API_PCBV2_NOT_NOT_VALID.createException(vseFileName, "No Network Entity Found.");
        }
        try {
            handleCollectionInfo(network);
        } catch (Exception e) {
            logger.warn("Error while getting CollectionInfoList from Network Entity", e);
            if (invoker != null) {
                invoker.logMsg("Unable to process collection information");
            }
        }
        if (devicesList == null || devicesList.isEmpty()) {
            throw PariAPIMessages._API_PCBV2_NOT_NOT_VALID.createException(vseFileName,
                    "No Device Entities Found.");
        }
        logger.info("Converting " + numDevs + " devices from VSEM format to PCBV2 format");
        if (numDevs == 0) {
            logger.info("No devices found to import from vsem: " + vseFileName + " . Exiting...");
            if (invoker != null) {
                invoker.logMsg("No device found to import. Exiting...");
            }
            return;
        }
        Map<Integer, EntityData> deviceDataMap = new LinkedHashMap<Integer, EntityData>();
        for (int idx = 0; idx < devices.length; idx++) {
            deviceDataMap.put(idx, devices[idx]);
        }
        // For Memory optimization purposes to scale better, remove the device Entity from the VSE Data
        devicesList.removeAllEntities();
        devices = null;
        int curDevIdx = 0;
        final Map<String/* ipAddr */, Long/* discoveredTime */> deviceDiscTimeMap = new ConcurrentHashMap<String, Long>();
        CustomerInstance custInstance = null;
        int instanceID = -1;
        if (customerId >= 0 && wingInstanceName != null && !wingInstanceName.isEmpty()) {
            try {
                custInstance = CustomerIfImpl.getInstance().getCustomerWingInstanceByName(customerId,
                        wingInstanceName);
            } catch (Exception e1) {
                logger.error(
                        "Error while getting instance id from customer: " + customerId + "/" + wingInstanceName,
                        e1);
            }
        }
        if (custInstance != null && custInstance.getInstanceId() != -1 && deviceDiscTimeMap.isEmpty()) {
            instanceID = custInstance.getInstanceId();
            Map<String, Long> tempMap = InventoryDBHelper.getDiscoveryTimeForAllDevices(customerId, instanceID);
            if (tempMap != null) {
                deviceDiscTimeMap.putAll(tempMap);
            }
        }

        try {

            try {
                /*
                 * split device data in to multiple sub arrays and process each array in separate thread.
                 */

                ArrayList<EntityData> devDataArray = new ArrayList<EntityData>();

                Iterator<Integer> i = deviceDataMap.keySet().iterator();

                while (i.hasNext()) {
                    final EntityData devData = deviceDataMap.get(i.next());
                    devDataArray.add(devData);
                    curDevIdx++;
                }

                if (devDataArray.size() < subArraySize) {
                    subArraySize = 10;
                }

                List<List<EntityData>> subArrayList = split(devDataArray, subArraySize);
                devDataArray.clear();

                logger.info("Sub Array size after splitting " + subArrayList.size());

                CountDownLatch latch = new CountDownLatch(subArrayList.size());

                for (List<EntityData> entityDataList : subArrayList) {
                    processDeviceArray(entityDataList, deviceDiscTimeMap, latch);
                    entityDataList.clear();

                }

                deviceDataMap.clear();
                deviceDiscTimeMap.clear();
                latch.await();

            } catch (Exception ex) {
                logger.error("Exception while importing device", ex);
            }

        } catch (Exception ex) {
            logger.error("Exception while importing devices from VSEM file", ex);
        }

    }

    /*
     * This method splits array in to sub array of size specified by 'len' argument
     */
    public static <T> List<List<T>> split(List<T> alist, int len) {
        List<List<T>> listOfLists = new ArrayList<>();
        int hi = 0;

        for (int lo = 0; lo < alist.size(); lo = hi) {
            hi = lo + len;
            if (hi > alist.size()) {
                hi = alist.size();
            }
            listOfLists.add(new ArrayList<T>(alist.subList(lo, hi)));
        }
        return listOfLists;
    }

    /*
     * Create device adder thread which accepts device list and executes Runnable job for further processing.
     */
    private void processDeviceArray(List<EntityData> subArray, Map<String, Long> deviceDiscTimeMap,
            CountDownLatch latch) {
        EntityData[] data = subArray.toArray(new EntityData[subArray.size()]);
        DeviceAdderThread daThread = new DeviceAdderThread(data, deviceDiscTimeMap, this, latch);

        executor.submit(daThread);
    }

    private void sanitizeProductModel(DSPNewDevice device) {
        // Some times, CSPC gives a wrong product model. Sanitize it and get it based on sysobject id in these cases.
        // Known cases of invalid CSPC product models
        // Empty
        // A
        // GenericCisco
        String prodModel = device.getProductId();
        prodModel = (prodModel == null) ? "" : prodModel.trim().toLowerCase();
        if (prodModel.isEmpty() || prodModel.equals("a") || prodModel.contains("generic")
                || prodModel.contains("ciscodevice") || prodModel.contains("catalyst")
                || prodModel.startsWith("wsc") // CSCug73786
                || prodModel.startsWith("cisco") || prodModel.startsWith("cev") || prodModel.contains("chas5=")) {
            String sysoid = device.getSysObjectId();
            if (sysoid != null && !sysoid.isEmpty()) {
                if (!sysoid.startsWith(".")) {
                    sysoid = "." + sysoid;
                }
                SNMPProductModelObject modelInfo = SNMPProductModelRepository.getInstance()
                        .getProductModelObjectFromId(sysoid);
                if (modelInfo != null) {
                    String realModel = modelInfo.getRealProductModel();
                    String model = modelInfo.getProductModel();
                    if (realModel != null) {
                        device.setProductId(realModel);
                    } else if (model != null) {
                        device.setProductId(model);
                    }

                }
            }
        }
    }

    private String addDeviceEntity(EntityData devData, Long discTime) throws Exception {
        PerDeviceImportStatus importDeviceStatus = new PerDeviceImportStatus();
        PerDeviceConfigBackupStatus deviceConfigBackupStatus = new PerDeviceConfigBackupStatus();
        long startTime = System.currentTimeMillis();
        importDeviceStatus.setStartTime(startTime);
        DSPNewDevice dev = new DSPNewDevice(devData);
        dev.setAttribute("DiscoveredTime", discTime);
        StringBuilder result = new StringBuilder();
        System.out.println("os = " + dev.getOsName());
        setOSTypeBasedOnSysDesc(dev);

        try {
            dspExecutor.execute(dev);
            Set<String> set = new HashSet<>();
            // Removing duplicates for SMUNames before storing in DB.
            String s = dev.getAttribute("SMUNames");
            if (s != null && !s.isEmpty()) {
                String[] al = s.split(",");
                for (String a : al) {
                    set.add(a);
                }
                s = set.toString().replace("[", "").replace("]", "");
                dev.setAttribute("SMUNames", s);
            }
            // Executes system and user defined DSP scripts
            DSPMetadataHandler metadataHandler = new DSPMetadataHandler();
            dev.setMetadataHandler(metadataHandler);
            String sysObjId = dev.getSysObjectId();
            if (sysObjId != null && sysObjId.length() != 0) {
                List<ScriptInfo> sysScripts = DeviceScriptManagerImpl.getInstance()
                        .getSystemDevicePackages(sysObjId, customerId);
                if (!sysScripts.isEmpty()) {
                    dspHandler.handlePackages(dev, sysScripts);
                }
                List<ScriptInfo> userScripts = DeviceScriptManagerImpl.getInstance().getUserDevicePackages(sysObjId,
                        customerId);
                if (!userScripts.isEmpty()) {
                    dspHandler.handlePackages(dev, userScripts);
                }

                // added as part if DSP ICF Integration
                List<ScriptInfo> icfScripts = DeviceScriptManagerImpl.getInstance().getIcfDevicePackages(sysObjId,
                        customerId);
                if (!icfScripts.isEmpty()) {
                    for (ScriptInfo ic : icfScripts) {
                        ICStatisticsManager.getInstance().populateICUsageStatisticsInfo(ic, 1);
                    }
                    dspHandler.handlePackages(dev, icfScripts);
                }
            } else {

                List<ScriptInfo> sysScripts = DeviceScriptManagerImpl.getInstance()
                        .getSystemDefinedZipPackages(PackageType.ZipInventory, "System");
                sysScripts = checkForApplicableScripts(sysScripts, dev);

                if (!sysScripts.isEmpty()) {
                    dspHandler.handlePackages(dev, sysScripts);
                }

                List<ScriptInfo> userScripts = DeviceScriptManagerImpl.getInstance()
                        .getUserDefinedZipPackages(PackageType.ZipInventory, customerId, "User");
                userScripts = checkForApplicableScripts(userScripts, dev);
                if (!userScripts.isEmpty()) {
                    dspHandler.handlePackages(dev, userScripts);
                }

                // added as part if DSP ICF Integration
                List<ScriptInfo> icfScripts = DeviceScriptManagerImpl.getInstance()
                        .getIcfDefinedZipPackages(PackageType.ZipInventory, customerId, "Icf");
                if (!icfScripts.isEmpty()) {
                    for (ScriptInfo ic : icfScripts) {
                        ICStatisticsManager.getInstance().populateICUsageStatisticsInfo(ic, 1);
                    }
                    dspHandler.handlePackages(dev, icfScripts);
                }
            }

        } catch (Exception ex) {
            logger.error("Excetption while executing DSP for device: " + dev.getDeviceName(), ex);
            result.append("<B>Exception while executing Device Support Scripts. Continuing anyway.</B><BR>");
            // We will continue and import the device as much as possible.
        }

        // Device coming as GenericDevice from CSPC and having running-config command other than show running-config,
        // running-config needs to be populated after setting OSType in DSP script
        // if (dev.getRunningConfig() == null || dev.getRunningConfig().isEmpty())

        //If device has running-config command other than show running-config, then populating data again.
        String command = InventoryDBHelper.getCommandForRunningConfig(dev.getOsName());
        if (command != null) {
            dev.populateConfigs();
        }

        sanitizeProductModel(dev); // CSCug64696 - Moved it after dspExecutor which parses sh versoin to get
        // ProductModel.

        importDeviceStatus.setHostName(dev.getDeviceName());
        importDeviceStatus.setIpAddress(dev.getIpAddress());
        importDeviceStatus.setOsName(dev.getOsName());
        importDeviceStatus.setOsVersion(dev.getVersionStr());
        importDeviceStatus.setDeviceFamily(dev.getDeviceFamily());
        importDeviceStatus.setProductModel(dev.getProductId());

        if (!dev.getColectionErrorList().isEmpty()) {
            if (perDeviceConfigBackupStatus == null) {
                perDeviceConfigBackupStatus = new ConcurrentHashMap<String, PerDeviceConfigBackupStatus>();
            }
            deviceConfigBackupStatus.setHostName(dev.getDeviceName());
            deviceConfigBackupStatus.setIpAddress(dev.getIpAddress());
            deviceConfigBackupStatus.setStartTime(startTime);
            deviceConfigBackupStatus.setEndTime(System.currentTimeMillis());
            deviceConfigBackupStatus.setErrorCollectionList(dev.getColectionErrorList());
            deviceConfigBackupStatus.setSuccess(false);
            perDeviceConfigBackupStatus.put(dev.getIpAddress(), deviceConfigBackupStatus);
        }

        // CSCtx24489 - Added grpName (group name) to the DeviceAdder
        DeviceAdder adder = new DeviceAdder(dev, InsertMode.update, customerId, wingInstanceName, rowId, invoker,
                login, result, collInfo, grpName, jobId, jobParameters);
        String str = adder.addDevice(null);
        importDeviceStatus.setEndTime(System.currentTimeMillis());
        List<DAVStatus> davStatus = this.davStatusData.get(importDeviceStatus.getIpAddress());
        StringBuilder davStatusMessage = new StringBuilder();
        boolean isSuccess = false;
        if (davStatus != null) {
            for (DAVStatus deviceStatus : davStatus) {
                if (deviceStatus.getStatus().equalsIgnoreCase("Successful")) {
                    importDeviceStatus.setDavStatus(deviceStatus.getStatus());
                    isSuccess = true;
                    continue;
                }
                davStatusMessage.append(deviceStatus.getProtocol() + " : " + deviceStatus.getStatus() + " <br>");
            }
            if (isSuccess) {
                importDeviceStatus.setDavStatusMessage(" - ");
            } else {
                importDeviceStatus.setDavStatus(" Failed ");
                importDeviceStatus.setDavStatusMessage(davStatusMessage.toString());
            }
        }

        if (str != null) {
            importDeviceStatus.setSuccess(false);
            importDeviceStatus.setMessage(str);
            perDeviceImportStatus.put(dev.getIpAddress(), importDeviceStatus);
            return str;
        }
        // adder.setConfigUpdated(true);
        DeviceStateChangeMessage sc = null;
        NetworkNode parent = adder.getNetworkNode();
        // If config updated for NXOS device, then we create checkpoint
        if (dev.getOsName().equalsIgnoreCase("nx-os") && adder.isConfigUpdated()) {
            nxosDevices.put(parent.getIpAddr().toString(), parent.getNodeId());
        }

        else if (dev.getOsName().equalsIgnoreCase("nxos") && adder.isConfigUpdated()) {
            nxosDevices.put(parent.getIpAddr().toString(), parent.getNodeId());
        }
        // If config updated for WLC device, then we collect 'transfer upload' config
        else if (dev.getOsName().equalsIgnoreCase("WLC") && adder.isConfigUpdated()) {
            wlcDevices = new ConcurrentHashMap<String, Integer>();
            wlcDevices.put(parent.getIpAddr().toString(), parent.getNodeId());
        } else if (dev.getOsName().equalsIgnoreCase("UCS")) {

            ucsDevices.put(parent.getIpAddr().toString(), parent.getNodeId());
        } else if (dev.getOsName().equalsIgnoreCase("ons")) {
            onsDevices.put("CPLoginUser", login);
            onsDevices.put(parent.getIpAddr().toString(), parent.getNodeId() + "");
        }

        else if (dev.getOsName().equalsIgnoreCase("JunOS") && adder.isConfigUpdated()) {

            junosDevices.put(parent.getIpAddr().toString(), parent.getNodeId());

        } else if (NmIfImpl.getInstance().isDeviceSupportedForResync(parent.getNodeId())
                && adder.isConfigUpdated()) {
            NetworkNode node = NetworkNodeCache.getInstance().getNodeByID(parent.getNodeId());

            DSPConfigData configData = executeConfigDSP(node);
            Map<String, String> details = new HashMap<String, String>();
            List<String> resyncConfigCmds = (List<String>) configData.getAttribute("resyncConfigCmds");
            if (resyncConfigCmds != null) {
                StringBuffer resyncCmd = new StringBuffer();
                for (String cmd : resyncConfigCmds) {
                    resyncCmd.append(cmd);
                    /**
                     * Added Space for handling file extensions in addon side
                     */
                    resyncCmd.append(" \n");
                }
                details.put("resyncCmd", resyncCmd.toString());
                details.put("waitTime", configData.getAttribute("WaitTime").toString());
                details.put("fileType", configData.getAttribute("FileType").toString());
                Object regex = configData.getAttribute("FileNameRegex");
                if (regex != null) {
                    details.put("regex", regex.toString());
                }
                otherDevices.put(parent.getIpAddr().toString(), details);
            }

        }
        importDeviceStatus.setSuccess(true);
        importDeviceStatus.setDeviceId(parent.getNodeId());
        perDeviceImportStatus.put(dev.getIpAddress(), importDeviceStatus);

        if (perDeviceConfigBackupStatus != null) {
            deviceConfigBackupStatus.setDeviceId(parent.getNodeId());
        }

        StringBuffer sb = new StringBuffer();

        Map<String, DSPNewDevice> modules = dev.getModules();
        deleteExistingModulesIfNecessary(modules, parent);
        int failedModules = addChildDevices(modules, parent, ModuleType.MODULE);

        Map<String, DSPNewDevice> newLogicalDevices = dev.getLogicalDevices();
        deleteExistingContextsIfNecessary(newLogicalDevices, parent, dev);
        int failedLogicalDevices = addChildDevices(newLogicalDevices, parent, ModuleType.CONTEXT);

        sc = new DeviceStateChangeMessage(parent.getNodeId(), true);
        ClientSessionManager.getInstance().sendMessageToAll(sc, parent.getNodeId(),
                ClientMessageType.NETWORK_NODE_RELATED);
        if (failedModules > 0) {
            // sb.append("Device successfully imported but " + failedModules + " Module" + ((failedModules > 1) ? "s" :
            // "") + " failed to import.");
            if (invoker != null) {
                invoker.logMsg(parent.getNodeName() + " imported successfully. But " + failedModules + " Module"
                        + ((failedModules > 1) ? "s" : "") + " failed to import.");
            }
        }
        if (failedLogicalDevices > 0) {
            // sb.append("Device successfully imported but " + failedLogicalDevices + " Context" +
            // ((failedLogicalDevices > 1) ? "s" : "") + " failed to import.");
            if (invoker != null) {
                invoker.logMsg(parent.getNodeName() + " imported successfully. But " + failedLogicalDevices
                        + " Context" + ((failedLogicalDevices > 1) ? "s" : "") + " failed to import.");
            }
        }
        if (sb.length() == 0) {
            return null;
        }
        return sb.toString();
    }

    private void setOSTypeBasedOnSysDesc(DSPNewDevice dev) {
        // Based on sysDescription, set the OS types
        ISnmpScalar sysDesc = dev.getSnmpScalar("1.3.6.1.2.1.1.1.0");
        String sysDescStr = sysDesc != null ? (String) sysDesc.getValue() : "";
        if (sysDescStr.toLowerCase().contains("staros")) {
            dev.setOSName("STAROS");
        } else if (sysDescStr.contains("IOS-XE")) {
            dev.setOSName("IOSXE");
        } else if (sysDescStr.contains("JUNOS")) {
            dev.setOSName("JunOS");
        }
    }

    private void publishNode(NetworkNode networkNode) {
        if (networkNode != null) {
            try {
                VirtualDeviceRefresh dRefresh = new VirtualDeviceRefresh(networkNode.getNodeId(), grpName, true, -1,
                        networkNode.getVersion().getVersionType(), false, customerId, wingInstanceName, false);
                dRefresh.setDeviceName(networkNode.getNodeName());
                if (invoker instanceof PcbImportJob) {
                    ((PcbImportJob) invoker).addVirtDeviceAddedMsg(dRefresh);
                } else {
                    try {
                        DeviceManager.getInstance().processDevice(MessageTypes.VIRUAL_DEVICE_DISCOVERED, dRefresh);
                    } catch (Exception ex) {
                        logger.error("Exception while processing virtual device add message", ex);
                    }
                    Messenger.getInstance().publish(MessageTypes.DEVICE_CONFIG_CHANGED,
                            networkNode.getNodeId() + "");
                }
            } catch (Exception ex) {
                logger.error("Exception while publishing final events for device: " + networkNode.getNodeName(),
                        ex);
            }
        }
    }

    /**
     * CSCtz28800 If the existing logical devices is null, then add all the new logical devices Else if any context is
     * missing in new logical devices from existing, then unmanage that context If new logical devices is null due to CP
     * not executed then we don't unmanage existing contexts If new logical devices is null due to context support is
     * not there , then we unmanage all existing contexts
     *
     * This method handles both deletion of contexts if required and publishing the contexts if context info was not
     * collected while re importing the already existing device with contexts.
     */
    private void deleteExistingContextsIfNecessary(Map<String, DSPNewDevice> newLogicalDevices, NetworkNode parent,
            DSPNewDevice dev) {

        List<ModuleInfo> existingLogicalDevices = InventoryDBHelper.getChildModulesOrContexts(parent.getNodeId(),
                ModuleType.CONTEXT);
        if (existingLogicalDevices != null) {
            if (newLogicalDevices != null) {
                logger.debug("existingLogicalDevices: " + existingLogicalDevices.toString());
                logger.debug("logicalDevices new: " + newLogicalDevices.keySet());
                Set<String> contextNames = newLogicalDevices.keySet();
                for (ModuleInfo modInfo : existingLogicalDevices) {
                    if (!contextNames.contains(modInfo.getSlotNumber())) // This context doesn't exist
                    {
                        try {
                            logger.debug("Unmanaging the context : " + modInfo.getSlotNumber() + "\t"
                                    + modInfo.getModId());
                            NmIfImpl.getInstance().unManageDevices(new int[] { modInfo.getModId() }, true, false);
                            parent.removeSupportedLogicalDeviceIds(modInfo.getSlotNumber());
                            InventoryDBHelper.deleteModuleInfo(modInfo.getModId(), parent.getNodeId(),
                                    modInfo.getSlotNumber(), ModuleType.CONTEXT.moduleType);
                        } catch (Exception e) {
                            logger.error("Error while unmanaging context : " + modInfo.getModId(), e);
                        }
                    }
                }
            } else if (dev.getContextSupported() != null && dev.getContextSupported().equalsIgnoreCase("no")) {
                for (ModuleInfo modInfo : existingLogicalDevices) {
                    try {
                        logger.debug(
                                "Unmanaging the context : " + modInfo.getSlotNumber() + "\t" + modInfo.getModId());
                        NmIfImpl.getInstance().unManageDevices(new int[] { modInfo.getModId() }, true, false);
                        parent.removeSupportedLogicalDeviceIds(modInfo.getSlotNumber());
                        InventoryDBHelper.deleteModuleInfo(modInfo.getModId(), parent.getNodeId(),
                                modInfo.getSlotNumber(), ModuleType.CONTEXT.moduleType);
                    } catch (Exception e) {
                        logger.error("Error while unmanaging context : " + modInfo.getModId(), e);
                    }
                }
            } else
            // Need to add the contexts again as it gets deleted from NetworkNodeCache.
            {
                for (ModuleInfo modInfo : existingLogicalDevices) {
                    NetworkNode networkNode = NetworkNodeCache.getInstance().getNodeByID(modInfo.getModId());
                    publishNode(networkNode);
                }
            }
        }
    }

    /**
     * If the existing modules is null, then add all the new modules. Else if entity mib data is not collected for that
     * device, then we don't delete any modules If entity mib data is present and existing module is not present in that
     * at the same slot and also not present in newModules then we unmanage that module.
     *
     * This method handles both deletion of modules if required and publishing the modules if module info was not
     * collected while re importing the already existing device with modules.
     */
    private void deleteExistingModulesIfNecessary(Map<String, DSPNewDevice> modules, NetworkNode parent) {
        boolean entityDataCollected = InventoryDBHelper.isEntityDataCollectedForDevice(parent.getNodeId());
        List<ModuleInfo> existingModules = InventoryDBHelper.getChildModulesOrContexts(parent.getNodeId(),
                ModuleType.MODULE);
        // if (existingModules != null && !existingModules.isEmpty() && modules != null)
        if (existingModules != null && !existingModules.isEmpty()) {
            Set<String> newModules;
            if (modules != null) {
                newModules = modules.keySet();
            } else {
                newModules = new HashSet<String>();
            }
            for (ModuleInfo modInfo : existingModules) {
                DeviceModule devMod = InventoryDBHelper.getDeviceModuleData(parent.getNodeId(),
                        modInfo.getSlotNumber());

                if (entityDataCollected && devMod == null && !newModules.contains(modInfo.getSlotNumber())) {
                    try {
                        NmIfImpl.getInstance().unManageDevices(new int[] { modInfo.getModId() }, true, false);
                        parent.removeSupportedModuleNodeId(modInfo.getSlotNumber());
                        InventoryDBHelper.deleteModuleInfo(modInfo.getModId(), parent.getNodeId(),
                                modInfo.getSlotNumber(), ModuleType.MODULE.moduleType);
                    } catch (Exception e) {
                        logger.error("Error while unmanaging module : " + modInfo.getModId(), e);
                    }
                }
                // CSCty94364 - when device is re-imported, device and its modules/contexts gets deleted and added to
                // NetworkNodeCache.
                // If module info was not collected as part of VSEM, we need to add the modules also when device gets
                // re-imported.
                // Otherwise, module gets removed from device tree.
                else {
                    NetworkNode moduleNode = NetworkNodeCache.getInstance().getNodeByID(modInfo.getModId());
                    publishNode(moduleNode);

                    // CSCud22202 - For each module , need to refresh its contexts for Module's context support
                    List<ModuleInfo> existingLogicalDevices = InventoryDBHelper
                            .getChildModulesOrContexts(moduleNode.getNodeId(), ModuleType.CONTEXT);
                    if (existingLogicalDevices != null) {
                        for (ModuleInfo contextInfo : existingLogicalDevices) {
                            NetworkNode modContextNode = NetworkNodeCache.getInstance()
                                    .getNodeByID(contextInfo.getModId());
                            publishNode(modContextNode);
                        }
                    }
                }
            }
        }

        // CSCtz31907 - Setting module's serial number from Entity MIB data if its present.
        if (modules != null) {
            for (String slot : modules.keySet()) {
                DeviceModule devMod = InventoryDBHelper.getDeviceModuleData(parent.getNodeId(), slot);
                if (devMod != null && devMod.getSerialNumber() != null && !devMod.getSerialNumber().isEmpty()) {
                    modules.get(slot).setSerialNumber(devMod.getSerialNumber());
                }
            }
        }
    }

    public int addChildDevices(Map<String, DSPNewDevice> devices, NetworkNode parent, ModuleType type) {
        int failedDevices = 0;
        DSPNewDevice dev;
        if (devices != null) {
            for (String context : devices.keySet()) {
                dev = devices.get(context);
                if (type == ModuleType.CONTEXT && parent != null) {
                    dev.setOSName(parent.getVersion().getVersionTypeStr());
                    String contextName = parent.getNodeName() + "/" + context;
                    dev.setDeviceName(contextName);
                    dev.setVersionStr(parent.getVersion().getVersionStr());
                    dev.setProductId(parent.getVersion().getProductId());
                    dev.setSerialNumber(parent.getVersion().getSerialNumber());
                }
                DeviceAdder adder = new DeviceAdder(dev, InsertMode.update, customerId, wingInstanceName, rowId,
                        invoker, login, new StringBuilder(), collInfo, grpName, jobId, jobParameters);
                adder.setSlot(context);
                adder.setParent(parent);
                try {
                    dspExecutor.execute(dev);
                    // Executes system and user defined DSP scripts
                    DSPMetadataHandler metadataHandler = new DSPMetadataHandler();
                    dev.setMetadataHandler(metadataHandler);
                    String sysObjId = dev.getSysObjectId();
                    if (sysObjId != null && sysObjId.length() != 0) {
                        List<ScriptInfo> sysScripts = DeviceScriptManagerImpl.getInstance()
                                .getSystemDevicePackages(sysObjId, customerId);
                        if (!sysScripts.isEmpty()) {
                            dspHandler.handlePackages(dev, sysScripts);
                        }
                        List<ScriptInfo> userScripts = DeviceScriptManagerImpl.getInstance()
                                .getUserDevicePackages(sysObjId, customerId);
                        if (!userScripts.isEmpty()) {
                            dspHandler.handlePackages(dev, userScripts);
                        }

                        // added as part if DSP ICF Integration
                        List<ScriptInfo> icfScripts = DeviceScriptManagerImpl.getInstance()
                                .getIcfDevicePackages(sysObjId, customerId);
                        if (!icfScripts.isEmpty()) {
                            for (ScriptInfo ic : icfScripts) {
                                ICStatisticsManager.getInstance().populateICUsageStatisticsInfo(ic, 1);
                            }
                            dspHandler.handlePackages(dev, icfScripts);
                        }
                    } else {

                        List<ScriptInfo> sysScripts = DeviceScriptManagerImpl.getInstance()
                                .getSystemDefinedZipPackages(PackageType.ZipInventory, "System");
                        sysScripts = checkForApplicableScripts(sysScripts, dev);

                        if (!sysScripts.isEmpty()) {
                            dspHandler.handlePackages(dev, sysScripts);
                        }

                        List<ScriptInfo> userScripts = DeviceScriptManagerImpl.getInstance()
                                .getUserDefinedZipPackages(PackageType.ZipInventory, customerId, "User");
                        userScripts = checkForApplicableScripts(userScripts, dev);
                        if (!userScripts.isEmpty()) {
                            dspHandler.handlePackages(dev, userScripts);
                        }

                        // added as part if DSP ICF Integration
                        List<ScriptInfo> icfScripts = DeviceScriptManagerImpl.getInstance()
                                .getIcfDefinedZipPackages(PackageType.ZipInventory, customerId, "Icf");
                        if (!icfScripts.isEmpty()) {
                            for (ScriptInfo ic : icfScripts) {
                                ICStatisticsManager.getInstance().populateICUsageStatisticsInfo(ic, 1);
                            }
                            dspHandler.handlePackages(dev, icfScripts);
                        }
                    }
                } catch (Exception e) {
                    logger.error("Excetption while executing DSP for device: " + dev.getDeviceName(), e);
                }
                String str = adder.addDevice(null);
                if (str == null) {
                    NetworkNode child = adder.getNetworkNode();
                    if (child != null && parent != null) {
                        int childId = child.getNodeId();
                        int parentId = parent.getNodeId();
                        if (type == ModuleType.MODULE) {
                            parent.insertSupportedModuleNodeId(context, childId);
                        } else {
                            parent.insertSupportedLogicalDeviceIds(context, childId);
                        }
                        InventoryDBHelper.insertModuleInfo(childId, parentId, context, type.moduleType);
                        // Add This module to the parent
                        // Add this parent to the module.
                        ModuleInfo mInfo = new ModuleInfo();
                        mInfo.setParentId(parentId);
                        mInfo.setSlotNumber(context);
                        mInfo.setType(type);
                        child.setSupportedModuleInfo(mInfo);
                        // Set child node DAV status same as parent.
                        child.setDeviceMode(parent.getDeviceMode());
                        child.setTransport(parent.getTransport());
                        child.setCredSetName(parent.getCredSetName());
                        DeviceStateChangeMessage sc = new DeviceStateChangeMessage(child.getNodeId(), true);
                        ClientSessionManager.getInstance().sendMessageToAll(sc, child.getNodeId(),
                                ClientMessageType.NETWORK_NODE_RELATED);
                    }

                    // Adding Module's contexts if present.
                    Map<String, DSPNewDevice> logicalDevices = dev.getLogicalDevices();
                    deleteExistingContextsIfNecessary(logicalDevices, child, dev); // CSCud22202
                    int failedLogicalDevices = addChildDevices(logicalDevices, child, ModuleType.CONTEXT);
                    if (failedLogicalDevices > 0 && invoker != null) {
                        invoker.logMsg("Unable to import " + failedLogicalDevices + " logical devices :" + str);
                    }
                } else {
                    if (invoker != null) {
                        if (type == ModuleType.MODULE) {
                            invoker.logMsg("Unable to import module at slot " + context + ":" + str);
                        } else {
                            invoker.logMsg("Unable to import context '" + context + "':" + str);
                        }
                    }
                    failedDevices++; // CSCty80345 - If context import fails
                }

            }
        }

        return failedDevices;
    }

    private void handleCollectionInfo(EntityData network) {
        if (invoker == null) {
            return;
        }
        try {
            EntityVector collInfoList = (EntityVector) network.getPropValue("CollectionInfoList");
            if (collInfoList != null && !collInfoList.isEmpty()) {
                EntityData collEntity = collInfoList.getEntities()[0];
                EntityVector logEV = (EntityVector) collEntity.getPropValue("LogList");
                collInfo = readCollectionInfo(collEntity, logEV);
                if (collInfo != null) {
                    invoker.logMsg("Collection Profile " + collInfo.getTitle() + " collected and imported");
                    invoker.logMsg("Collection Profile started at " + format.format(collInfo.getStartTime())
                            + ", ended at " + format.format(collInfo.getEndTime()));
                    List<VSEMCollectionLog> logList = collInfo.getLogList();
                    if (logList != null && !logList.isEmpty()) {
                        for (VSEMCollectionLog log : logList) {
                            if (log != null) {
                                invoker.logMsg(log.getMsg());
                            }
                        }
                    }
                }
            }
        } catch (Exception ex) {
            logger.error("Exception while handling collection info for VSE file: " + vseFileName, ex);
        }
    }

    private VSEFileProcessor getVSE(File pcb, String pcbFileName, File rootDir)
            throws IOException, PariAPIException {
        if (!pcb.isDirectory()) {
            logger.debug("Unzipping " + pcbFileName + " to " + rootDir.getAbsolutePath());
            FileInputStream fis = new FileInputStream(pcb);
            ZipInputStream zin = new ZipInputStream(fis);
            try {
                UnzipDir.unzip2(pcbFileName, rootDir.getAbsolutePath());
            } finally {
                if (fis != null) {
                    fis.close();
                }
                if (zin != null) {
                    zin.close();
                }
            }
            logger.debug("Done unzipping " + pcbFileName + " to " + rootDir.getAbsolutePath());
        }
        VSEFileProcessor fp = openVSEM(pcbFileName, rootDir);
        logger.debug("Done opening VSEM file.");
        return fp;
    }

    private VSEFileProcessor openVSEM(String pcbFilename, File rootDir) throws PariAPIException {
        EntityDefLoader.getInstance();
        EntityDefRepository defRepo = EntityDefRepository.getInstance();
        VSEFileProcessor fp = null;
        try {
            fp = new VSEFileProcessor(Format.XML, rootDir, defRepo);
        } catch (Exception ex) {
            logger.info("Exception while processing VSE File. " + pcbFilename, ex);
            throw PariAPIMessages._API_PCBV2_NOT_NOT_VALID.createException(pcbFilename, ex.getMessage());
        }
        fp.loadVSEFromFiles();
        return fp;
    }

    public static void main(String argv[]) throws Exception {
        BasicConfigurator.configure();
        VSEMImporter importer = new VSEMImporter(argv[0], -1, null, null, null, -1, "admin", 0,
                new JobParameters());
        importer.importVsem();
    }

    public int getNumberOfDevices() {
        return numDevs;
    }

    public Map<String, PerDeviceImportStatus> getPerDeviceImportStatus() {
        return this.perDeviceImportStatus;
    }

    public Map<String, PerDeviceConfigBackupStatus> getPerDeviceConfigBackupStatus() {
        return this.perDeviceConfigBackupStatus;
    }

    private class DeviceAdderThread implements Runnable {
        private VSEMImporter importer;
        private EntityData[] devDatas;
        private Map<String, Long> deviceDiscTimeMap;
        private CountDownLatch latch;

        public DeviceAdderThread(EntityData[] devDatas, Map<String, Long> deviceDiscTimeMap, VSEMImporter importer,
                CountDownLatch latch) {
            this.importer = importer;
            this.devDatas = devDatas;
            this.deviceDiscTimeMap = deviceDiscTimeMap;
            this.latch = latch;
        }

        @Override
        public void run() {
            for (EntityData devData : devDatas) {

                String ip = devData.getScalarPropValueAsString("IPAddress");
                String msg = null;
                String devName = devData.getScalarPropValueAsString("HostName");
                try {
                    String osType = devData.getScalarPropValueAsString("OSType");
                    if (devName == null || devName.isEmpty()) {
                        msg = "Device host name not given";
                    }
                    // CSCue09892 - If OSVersion is not coming from CSPC, it should be parsed from ShowVersion
                    // else if ( osVersion == null | osVersion.isEmpty())
                    // {
                    // msg = "OS Version name not given";
                    // }
                    else if (osType == null || osType.isEmpty()) {
                        msg = "OS Type is not given";
                    } else {
                        msg = addDeviceEntity(devData, deviceDiscTimeMap.get(ip));
                        devData.clearEntityData();
                    }
                    if (importer.invoker != null) {
                        if (msg == null) {
                            importer.invoker.logMsg("Successfully imported the device " + devName);
                            importer.numDevsDone.incrementAndGet();
                        } else {
                            importer.invoker.logMsg(msg);
                        }
                    }
                } catch (Exception e) {
                    logger.error("Excetption while adding device: " + ip, e);
                    if (importer.rowId > -1 && importer.customerId > -1) {
                        ServerDBHelper.insertPcbImportDetails(importer.customerId, importer.rowId, ip,
                                Constants.PCB_IMPORT_FAIL,
                                "<B>Exception while executing Device Support Scripts</B>");
                    }
                    msg = "Exception while adding device: " + devName + ". " + e.getMessage();
                } finally {

                    try {
                        if (importer.numDevsDone.get() % 100 == 0) {
                            logger.error("SIRI SIRI SIRI: " + importer.numDevsDone + " Done. Times: "
                                    + DeviceAdder.time1 + "," + DeviceAdder.time2 + "," + DeviceAdder.time3 + ","
                                    + DeviceAdder.time4 + "," + DeviceAdder.time5 + "," + DeviceAdder.time6);
                            System.err.println("SIRI SIRI SIRI: " + importer.numDevsDone + " Done. Times: "
                                    + DeviceAdder.time1 + "," + DeviceAdder.time2 + "," + DeviceAdder.time3 + ","
                                    + DeviceAdder.time4 + "," + DeviceAdder.time5 + "," + DeviceAdder.time6);
                        }

                        int perc = ((importer.numDevsDone.get() * 100) / importer.numDevs);
                        perc = Math.max(99, perc);
                        logger.debug(
                                perc + "% Done with " + importer.numDevsDone + "/" + importer.numDevs + " devices");
                        if (importer.invoker != null) {
                            boolean success = (msg == null);
                            if (msg == null) {
                                msg = "Successfully imported the device ";
                            }
                            HashMap<String, String> msgMap = new HashMap<String, String>();
                            msgMap.put(devName, msg);
                            importer.invoker.statusUpdate("", success, perc, msgMap);
                            Thread.sleep(50); // Give time to inform the client?
                        }
                    } catch (Exception ex) {
                        logger.error("Exception while updating device status", ex);
                    }
                }
            }
            // decrement count down latch
            latch.countDown();
        }
    }

    // This function will check if the node is valid for selected platform types.
    private boolean isPlatformApplicable(DSPNewDevice dev, final List<String> platformTypes) {
        for (String platformType : platformTypes) {
            String os = "_" + dev.getOsName();
            if (platformType.equals(os)) {
                return true;
            }
        }
        return false;
    }

    private List<ScriptInfo> checkForApplicableScripts(List<ScriptInfo> scripts, DSPNewDevice dev) {
        if (scripts != null) {
            DSPScriptCache cache = DSPScriptCache.getInstance();

            Iterator<ScriptInfo> itr = scripts.iterator();
            while (itr.hasNext()) {
                // Read platform type defined in script
                ScriptInfo script = itr.next();
                List<String> platformTypes = cache.getOSType(script.getIdentifier(), script.getCustomerId());
                // If platform types defined in script doesn't apply for device, we need to delete the script
                // from list. If it is null, we treat it as all platforms.
                if (platformTypes != null) {
                    if (!isPlatformApplicable(dev, platformTypes)) {
                        itr.remove();
                    }
                }
            }
            return scripts;
        }
        return scripts;

    }

    // Executes server side config dsp scripts that are applicable for given node.
    private DSPConfigData executeConfigDSP(NetworkNode node) {
        DSPConfigData configData = new DSPConfigData();
        try {
            DSPMetadataHandler metadataHandler = new DSPMetadataHandler();
            configData.setMetadataHandler(metadataHandler);
            configData.setAttribute("sysId", node.getSysObjectID());
            Set<Integer> deviceIds = new HashSet<>(Arrays.asList(node.getNodeId()));
            DSPHandler<DSPConfigData> handler = new ConfigResyncDSPHandler();
            // Get applicable System DSP packages for node and execute it.
            List<DeviceScriptInfo> sysScripts = DeviceScriptManagerImpl.getInstance().getScriptInfoList(-1, -1,
                    deviceIds, PackageType.Inventory, login, ScriptDefinition.System);
            if (!sysScripts.isEmpty()) {
                handler.handlePackages(configData, sysScripts);
            }
            // Get applicable User DSP packages for node and execute it.
            List<DeviceScriptInfo> userScripts = DeviceScriptManagerImpl.getInstance().getScriptInfoList(-1, -1,
                    deviceIds, PackageType.Inventory, login, ScriptDefinition.User);
            if (!userScripts.isEmpty()) {
                handler.handlePackages(configData, userScripts);
            }
            // Save DSP metadata to database..
            DSPMetadataHandler metaDataHandler = configData.getMetadataHandler();
            metaDataHandler.saveMetadata(node.getNodeId());
        } catch (Exception ex) {
            logger.error("Exception while executing config dsp script for node[ipaddress] = " + node.getNodeName()
                    + "[" + node.getIpAddr() + "]");
        }
        return configData;
    }
}