Java tutorial
// Copyright (c) 2016 Synopsys, Inc. All right reserved worldwide. package com.coverity.report.analysis; import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.util.*; import javax.ws.rs.NotAuthorizedException; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVPrinter; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import com.coverity.report.Personality; import com.coverity.report.analysis.jaxb.Report; import com.coverity.report.analysis.protecode.*; import com.coverity.report.common.ConnectionTesterUtils.ConnectionException; import com.coverity.report.common.Utils; import com.coverity.report.gui.Progress; import static com.coverity.report.analysis.jaxb.Report.Tools.ProtecodeSC.*; /** * @author vappidi. */ public class ProtecodeSCToolProcessor implements ToolProcessor { private static Logger log = LogManager.getLogger(); private Progress progress; private ProtecodeSCToolConfig config; private ProtecodeSCRestService restService; private List<DataFile> dataFiles = new ArrayList<>(); private List<ProtecodeSCComponentsData> componentsDataList; private List<ProtecodeSCVulnerabilitiesData> vulnerabilitiesDataList; private ProtecodeSCSystemInfo systemInfo = null; ProtecodeSCProductScan productScan = null; public ProtecodeSCToolProcessor(Progress progress) { this.progress = progress; } @Override public void configure(ToolConfig config) { this.config = (ProtecodeSCToolConfig) config; } @Override public void setProgress(Progress progress) { this.progress = progress; } @Override public boolean isEnabled() { return config.isEnabled(); } @Override public void connect() throws ToolProcessorException, ConnectionException { new ConnectionTester(config.serverUrl, config.username, config.getPassword(), getTool()).checkConnection(); restService = new ProtecodeSCRestService(config.serverUrl, config.username, config.getPassword()); try { restService.checkPassword(); } catch (NotAuthorizedException e) { throw new ToolAuthenticationException(e.getMessage()); } catch (Exception e) { log.debug(Utils.formatThrowable(e)); throw new ToolProcessorException(e.getMessage()); } } @Override public Map<String, String> enumerateScopes() throws ToolProcessorException { Map<String, String> map = new HashMap<>(); try { ProtecodeSCProductData productsData = restService.getProductsInfo(); List<ProtecodeSCProductData.Product> products = productsData.getProducts(); products.forEach(product -> map.put(product.getName(), String.valueOf(product.getId()))); } catch (NotAuthorizedException e) { throw new ToolAuthenticationException(e.getMessage()); } catch (Exception e) { log.debug(Utils.formatThrowable(e)); throw new ToolProcessorException(e.getMessage()); } return map; } @Override public List<String> validate() { List<String> errors = config.validate(); if (!errors.isEmpty()) { return errors; } Map<String, String> products = null; try { products = enumerateScopes(); } catch (Exception x) { errors.add(x.getMessage()); return errors; } if (errors.isEmpty()) { if (config.productId == null) { // Use the product name to get the product ID String productNameStr = config.productName; String productIdStr = products.get(productNameStr); if (productIdStr == null) { errors.add("Protecode SC product " + productNameStr + " no longer exists."); } else { try { config.productId = Long.parseLong(productIdStr); } catch (NumberFormatException x) { throw new RuntimeException( "Protecode SC product ID \"" + productIdStr + "\" is not a number"); } } } else { // Validate the productId by finding it in the list. if (!products.containsValue(config.productId.toString())) { errors.add("The Protecode SC product with ID " + config.productId + " no longer exists."); } } } return errors; } @Override public void readData() throws ToolProcessorException { componentsDataList = restService.getComponentsInfo(config.productId); vulnerabilitiesDataList = restService.getVulnerabilitiesInfo(config.productId); systemInfo = restService.getSystemInfo(); productScan = restService.getProductScanResult(config.productId); try { File componentsTempFile = File.createTempFile(Personality.getInstance().tempFileNameBase(), ".csv"); File vulnerabilitiesTempFile = File.createTempFile(Personality.getInstance().tempFileNameBase(), ".csv"); String[] componentsHeadings = new String[] { "component name", "version", "license", "license type", "vulnerability count", "object" }; try (FileWriter writer = new FileWriter(componentsTempFile)) { CSVPrinter printer = new CSVPrinter(writer, CSVFormat.EXCEL); printer.printRecord(componentsHeadings); componentsDataList.stream().forEach(info -> { String[] componentsRecord = new String[componentsHeadings.length]; componentsRecord[0] = info.getComponent(); componentsRecord[1] = info.getVersion(); componentsRecord[2] = info.getLicense(); componentsRecord[3] = info.getLicenseType(); componentsRecord[4] = String.valueOf(info.getVulnerabilityCount()); componentsRecord[5] = info.getObject(); try { printer.printRecord(componentsRecord); } catch (Exception ignored) { } }); dataFiles.add(new DataFile("protecode-components.csv", componentsTempFile)); } catch (IOException e) { throw makeException("Could not write protecode components data to CSV file " + componentsTempFile.getAbsolutePath(), e); } String[] vulnerabilitiesHeadings = new String[] { "component", "version", "CVE", "CVSS" }; try (FileWriter writer = new FileWriter(vulnerabilitiesTempFile)) { CSVPrinter printer = new CSVPrinter(writer, CSVFormat.EXCEL); printer.printRecord(vulnerabilitiesHeadings); vulnerabilitiesDataList.stream().forEach(info -> { String[] vulnerabilitiesRecord = new String[vulnerabilitiesHeadings.length]; vulnerabilitiesRecord[0] = info.getComponent(); vulnerabilitiesRecord[1] = info.getVersion(); vulnerabilitiesRecord[2] = info.getCve(); vulnerabilitiesRecord[3] = String.valueOf(info.getCvss()); try { printer.printRecord(vulnerabilitiesRecord); } catch (Exception ignored) { } }); dataFiles.add(new DataFile("protecode-vulnerabilities.csv", vulnerabilitiesTempFile)); } catch (IOException e) { throw makeException("Could not write protecode vulnerabilities to CSV file " + vulnerabilitiesTempFile.getAbsolutePath(), e); } } catch (IOException e) { throw makeException("Cannot create temporary file", e); } } private String makeKey(String name, String version) { if (version.isEmpty()) { return name; } else { return name + " " + version; } } private enum Criticality { Critical, Major, Minor; static Criticality fromScore(float score) { if (score >= 7) { return Critical; } else if (score >= 4) { return Major; } else { return Minor; } } } private static class ComponentCriticalities { String name; Map<Criticality, Integer> map; ComponentCriticalities(String name, Map<Criticality, Integer> map) { this.name = name; this.map = map; } int sum() { return map.values().stream().mapToInt(Integer::intValue).sum(); } } @Override public void getAggregatedData(Report report) { Report.Tools.ProtecodeSC protecodeSC = new Report.Tools.ProtecodeSC(); report.getTools().setProtecodeSC(protecodeSC); // Collapse the components data by removing non-unique entries. This is needed in // order to make the results match what the Protecode SC GUI shows. Map<String, ProtecodeSCComponentsData> uniqueComponentsMap = new TreeMap<>(); componentsDataList.stream() .forEach(data -> uniqueComponentsMap.put(data.getComponent() + "-" + data.getVersion(), data)); // Similarly, collapse the vulnerabilities data using the name, version and CVE of each vulnerability // as the key Map<String, ProtecodeSCVulnerabilitiesData> uniqueVulnerabilitiesMap = new TreeMap<>(); vulnerabilitiesDataList.stream().forEach(data -> uniqueVulnerabilitiesMap .put(data.getComponent() + "-" + data.getVersion() + "-" + data.getCve(), data)); // Make a map for counting the criticalities int knownVulns = uniqueVulnerabilitiesMap.size(); int vulnerableComponents = 0, totCriticalVulns = 0, totMajorVulns = 0, totMinorVulns = 0; Map<String, Map<Criticality, Integer>> componentVulnerabilityMap = new HashMap<>(); // Fill the map with zeroes uniqueComponentsMap.values().stream().forEach(comp -> { Map<Criticality, Integer> severityMap = new HashMap<>(); Arrays.stream(Criticality.values()).forEach(c -> severityMap.put(c, 0)); componentVulnerabilityMap.put(makeKey(comp.getComponent(), comp.getVersion()), severityMap); }); // Accumulate the count for each criticality level for (ProtecodeSCVulnerabilitiesData vulnerabilitiesData : uniqueVulnerabilitiesMap.values()) { Criticality criticality = Criticality.fromScore(vulnerabilitiesData.getCvss()); Map<Criticality, Integer> severityMap = componentVulnerabilityMap .get(makeKey(vulnerabilitiesData.getComponent(), vulnerabilitiesData.getVersion())); severityMap.put(criticality, severityMap.get(criticality) + 1); } for (Map.Entry<String, Map<Criticality, Integer>> map : componentVulnerabilityMap.entrySet()) { totCriticalVulns += map.getValue().get(Criticality.Critical); totMajorVulns += map.getValue().get(Criticality.Major); totMinorVulns += map.getValue().get(Criticality.Minor); } // Count the number of licenses of each type. // First collapse the list of components by license Map<String, String> licenseTypeMap = new TreeMap<>(); uniqueComponentsMap.values().forEach(c -> licenseTypeMap.put(c.getLicense(), c.getLicenseType())); // Then count the number of each type. Map<String, Integer> licenseTypeCountMap = new HashMap<>(); licenseTypeMap.values().forEach(type -> { Integer licenseCount = licenseTypeCountMap.get(type); if (licenseCount == null) { licenseCount = 1; } else { ++licenseCount; } licenseTypeCountMap.put(type, licenseCount); }); { // Policy elements for (ProtecodeSCComponentsData componentsData : uniqueComponentsMap.values()) { if (componentsData.getVulnerabilityCount() > 0) { vulnerableComponents++; } } protecodeSC.setPolicyElements(new PolicyElements()); List<PolicyElements.Point> policyElements = protecodeSC.getPolicyElements().getPoint(); policyElements.add(makePolicyElement(1, "Components with Vulnerabilities", vulnerableComponents)); policyElements.add(makePolicyElement(2, "Known Vulnerabilities", knownVulns)); policyElements.add(makePolicyElement(3, "Critical Vulnerabilities", totCriticalVulns)); } { // Components int totalComponents = 0, noKnownVulnerablitiesCount = 0; totalComponents = uniqueComponentsMap.size(); noKnownVulnerablitiesCount = totalComponents - vulnerableComponents; protecodeSC.setComponents(new Components()); List<Components.Point> components = protecodeSC.getComponents().getPoint(); components.add(makeComponents(1, "Vulnerable", vulnerableComponents)); components.add(makeComponents(2, "No known vulnerabilities", noKnownVulnerablitiesCount)); } { // Vulnerabilities protecodeSC.setVulnerabilities(new Vulnerabilities()); List<Vulnerabilities.Point> vulnerabilities = protecodeSC.getVulnerabilities().getPoint(); vulnerabilities.add(makeVulnerabilities(1, "Critical", totCriticalVulns)); vulnerabilities.add(makeVulnerabilities(2, "Major", totMajorVulns)); vulnerabilities.add(makeVulnerabilities(3, "Minor", totMinorVulns)); } { // Licenses protecodeSC.setLicenses(new Licenses()); List<Licenses.Point> licenses = protecodeSC.getLicenses().getPoint(); int order = 0; for (Map.Entry<String, Integer> entry : licenseTypeCountMap.entrySet()) { Licenses.Point point = new Licenses.Point(); point.setCount(entry.getValue()); point.setLabel(entry.getKey()); point.setOrder(order++); licenses.add(point); } } { // Top 10 Vulnerable Components // Roll up final int ROLLUP_INDEX = 10; List<ComponentCriticalities> pairs = makeRolledUpList(componentVulnerabilityMap, ROLLUP_INDEX); // Transform to a list of points int order = pairs.size(); protecodeSC.setTop10VulnerableComponents(new Top10VulnerableComponents()); List<Top10VulnerableComponents.Component> top10VulnerableComponents = protecodeSC .getTop10VulnerableComponents().getComponent(); for (ComponentCriticalities pair : pairs) { Top10VulnerableComponents.Component component = new Top10VulnerableComponents.Component(); component.setOrder(order--); component.setLabel(pair.name); List<Top10VulnerableComponents.Component.Point> points = component.getPoint(); pair.map.forEach((k, v) -> { Top10VulnerableComponents.Component.Point point = new Top10VulnerableComponents.Component.Point(); point.setCount(v); point.setSeverity(k.name()); points.add(point); }); top10VulnerableComponents.add(component); } } } @Override public List<DataFile> getDetailedDataFiles() { return dataFiles; } @Override public List<String> getConfigurationText() throws ToolProcessorException { List<String> configLines = new ArrayList<>(); if (systemInfo == null || productScan == null) { configLines.add("We couldn't gather configuration information. Please use a more up-to-date Appcheck."); } else { String lastUpdatedScanDate = Utils.longDateFormat(productScan.getResults().getLast_updated()); configLines.add("Scan Date and Time: " + lastUpdatedScanDate); configLines.add("CVE Database Date: " + Utils.longDateFormat(systemInfo.getStatus().getVersion().getCve_database())); configLines.add("Frontend Version: " + systemInfo.getStatus().getVersion().getFrontend()); configLines.add("Worker Version: " + systemInfo.getStatus().getVersion().getWorker()); } return configLines; } @Override public Tool getTool() { return Tool.ProtecodeSC; } private static ToolProcessorException makeException(String message, Exception e) { return new ToolProcessorException("Protecode: " + message + ": " + e.getMessage(), e); } private PolicyElements.Point makePolicyElement(int order, String label, int count) { PolicyElements.Point point = new PolicyElements.Point(); point.setCount(count); point.setLabel(label); point.setOrder(order); return point; } private Components.Point makeComponents(int order, String label, int count) { Components.Point point = new Components.Point(); point.setCount(count); point.setLabel(label); point.setOrder(order); return point; } private Vulnerabilities.Point makeVulnerabilities(int order, String label, int count) { Vulnerabilities.Point point = new Vulnerabilities.Point(); point.setCount(count); point.setLabel(label); point.setOrder(order); return point; } private static class CountPair<T> { public T key; public Integer count; CountPair(T key, Integer count) { this.key = key; this.count = count; } } private List<ComponentCriticalities> makeRolledUpList(Map<String, Map<Criticality, Integer>> map, final int rollupIndex) { // Transfer the data to a list List<ComponentCriticalities> pairs = new ArrayList<>(); map.forEach((k, v) -> pairs.add(new ComponentCriticalities(k, v))); // Sort by sum pairs.sort((a, b) -> b.sum() - a.sum()); // roll up the last n - rollupIndex pairs into another pair ComponentCriticalities other = new ComponentCriticalities("Other Components", new HashMap<>()); Arrays.stream(Criticality.values()).forEach(c -> other.map.put(c, 0)); for (int i = pairs.size() - 1; i >= rollupIndex; --i) { pairs.get(i).map.forEach((k, v) -> other.map.put(k, other.map.get(k) + v)); pairs.remove(i); } // If other is nonempty, add it. if (other.sum() > 0) { pairs.add(other); } return pairs; } }