Java tutorial
//////////////////////////////////////////////////////////////////////// // // Copyright (c) 2009-2015 Denim Group, Ltd. // // The contents of this file are subject to the Mozilla Public License // Version 2.0 (the "License"); you may not use this file except in // compliance with the License. You may obtain a copy of the License at // http://www.mozilla.org/MPL/ // // Software distributed under the License is distributed on an "AS IS" // basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the // License for the specific language governing rights and limitations // under the License. // // The Original Code is ThreadFix. // // The Initial Developer of the Original Code is Denim Group, Ltd. // Portions created by Denim Group, Ltd. are Copyright (C) // Denim Group, Ltd. All Rights Reserved. // // Contributor(s): Denim Group, Ltd. // //////////////////////////////////////////////////////////////////////// package com.denimgroup.threadfix.importer.impl.upload; import com.denimgroup.threadfix.annotations.ScanFormat; import com.denimgroup.threadfix.annotations.ScanImporter; import com.denimgroup.threadfix.data.ScanCheckResultBean; import com.denimgroup.threadfix.data.ScanImportStatus; import com.denimgroup.threadfix.data.entities.*; import com.denimgroup.threadfix.importer.exception.ScanFileUnavailableException; import com.denimgroup.threadfix.importer.impl.AbstractChannelImporter; import com.denimgroup.threadfix.importer.util.DateUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.transaction.annotation.Transactional; import javax.annotation.Nonnull; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.*; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import static com.denimgroup.threadfix.CollectionUtils.list; import static com.denimgroup.threadfix.CollectionUtils.map; /** * Created by mhatzenbuehler on 8/4/2014. */ @ScanImporter(scannerName = ScannerDatabaseNames.CLANG_DB_NAME, format = ScanFormat.ZIP) public class ClangChannelImporter extends AbstractChannelImporter { public ClangChannelImporter() { super(ScannerType.CLANG); } private static final String BUGTAIL = " -->"; private static final String BUGDESC = "<!-- BUGDESC "; private static final String BUGTYPE = "<!-- BUGTYPE "; private static final String BUGCATEGORY = "<!-- BUGCATEGORY "; private static final String BUGPATH = "<!-- BUGFILE "; private static final String BUGLINE = "<!-- BUGLINE "; private static final String BUGCOLUMN = "<!-- BUGCOLUMN "; private static final String BUGFILE_START = "<tr><td class=\"rowname\">File:</td><td>"; private static final String BUGFILE_END = "</td></tr>"; private static final String REGEX_LINE_SOURCE = "<tr><td class=\"num\" id=\"LN[0-9]+\">[0-9]+</td><td class=\"line\">.*"; private static final String REGEX_LINE_COMMENT = "<tr><td class=\"num\"></td><td class=\"line\"><div id=\"(End)*Path\\d*\" class=\"msg.*"; private static final String REGEX_REPORT_FILE = ".*/report-[0-9a-f]{6}.html"; @Override @Transactional public Scan parseInput() { zipFile = unpackZipStream(); Scan scan = createScanWithFileNames(); scan.setImportTime(getImportTime()); Map<String, InputStream> reports = getReportFiles(); List<Finding> findings = list(); for (Map.Entry<String, InputStream> entry : reports.entrySet()) { findings.add(parseInputStream(entry.getKey(), entry.getValue())); } scan.setFindings(findings); deleteZipFile(); return scan; } private Finding parseInputStream(String reportFileName, InputStream in) { List<DataFlowElement> dataFlowElements = list(); Map<FindingKey, String> findingKeyStringMap = map(); String bugDesc = null; String bugType = null; String bugCategory = null; String bugPath = null; String bugFile = null; String bugLine = null; String bugColumn = null; String line, nonHtmlLine; String previousSourceLine = ""; String previousLineNumber = ""; boolean foundLastError = false; int dataFlowSeq = 0; BufferedReader reader = new BufferedReader(new InputStreamReader(in)); try { line = reader.readLine(); while (line != null && !foundLastError) { if (line.startsWith(BUGDESC) && line.endsWith(BUGTAIL)) bugDesc = StringUtils.substringBetween(line, BUGDESC, BUGTAIL); else if (line.startsWith(BUGTYPE) && line.endsWith(BUGTAIL)) bugType = StringUtils.substringBetween(line, BUGTYPE, BUGTAIL); else if (line.startsWith(BUGCATEGORY) && line.endsWith(BUGTAIL)) bugCategory = StringUtils.substringBetween(line, BUGCATEGORY, BUGTAIL); else if (line.startsWith(BUGPATH) && line.endsWith(BUGTAIL)) bugPath = StringUtils.substringBetween(line, BUGPATH, BUGTAIL); else if (line.startsWith(BUGLINE) && line.endsWith(BUGTAIL)) bugLine = StringUtils.substringBetween(line, BUGLINE, BUGTAIL); else if (line.startsWith(BUGCOLUMN) && line.endsWith(BUGTAIL)) bugColumn = StringUtils.substringBetween(line, BUGCOLUMN, BUGTAIL); else if (line.startsWith(BUGFILE_START) && line.endsWith(BUGFILE_END)) bugFile = StringUtils.substringBetween(line, BUGFILE_START, BUGFILE_END); if (line.matches(REGEX_LINE_SOURCE)) { nonHtmlLine = line.replaceAll("<span class='expansion'>([^<]*)</span>", ""); nonHtmlLine = nonHtmlLine.replaceAll("<[^>]*>", ""); // strip html previousLineNumber = StringUtils.substringBetween(line, "id=\"LN", "\">"); previousSourceLine = nonHtmlLine.replaceFirst(previousLineNumber, ""); } else if (line.matches(REGEX_LINE_COMMENT)) { DataFlowElement element = new DataFlowElement(); element.setLineText(previousSourceLine); element.setSourceFileName(bugFile); if (line.contains("id=\"EndPath\"")) { element.setLineNumber(Integer.parseInt(bugLine)); element.setColumnNumber(Integer.parseInt(bugColumn)); foundLastError = true; } else { element.setLineNumber(Integer.parseInt(previousLineNumber)); } String prevElemLineText = null; if (dataFlowSeq > 0) { prevElemLineText = dataFlowElements.get(dataFlowSeq - 1).getLineText(); } if (prevElemLineText != null && element.getLineText().equals(prevElemLineText)) { // concurrent line comments, overwrite last element element.setSequence(dataFlowSeq); dataFlowElements.set(dataFlowSeq - 1, element); } else { element.setSequence(++dataFlowSeq); dataFlowElements.add(element); } } line = reader.readLine(); } reader.close(); } catch (IOException e) { log.error("IOException thrown when reading file " + reportFileName, e); } String vulnCode = bugCategory.concat(":").concat(bugType); findingKeyStringMap.put(FindingKey.VULN_CODE, vulnCode); findingKeyStringMap.put(FindingKey.DETAIL, bugDesc); findingKeyStringMap.put(FindingKey.PATH, bugFile); findingKeyStringMap.put(FindingKey.SEVERITY_CODE, "Medium"); Finding finding = super.constructFinding(findingKeyStringMap); if (finding == null) { throw new IllegalStateException("XML was invalid or we didn't parse out enough information"); } finding.setIsStatic(true); finding.setDataFlowElements(dataFlowElements); finding.setSourceFileLocation(bugPath); finding.setNativeId(getNativeIdFromReport(reportFileName)); return finding; } private String getNativeIdFromReport(String input) { String id; id = StringUtils.substringBetween(input, "report-", ".html"); if (id == null) id = StringUtils.substringAfterLast(input, "/"); if (id == null) id = input; if (id.length() > 50) id = id.substring(id.length() - 50); return id; } private Map<String, InputStream> getReportFiles() { if (zipFile.entries() == null) { throw new ScanFileUnavailableException("No zip entries were found in the zip file."); } Map<String, InputStream> m = map(); Enumeration<? extends ZipEntry> entries = zipFile.entries(); while (entries.hasMoreElements()) { ZipEntry entry = entries.nextElement(); if (entry.getName().matches(REGEX_REPORT_FILE)) { try { m.put(entry.getName(), zipFile.getInputStream(entry)); } catch (IOException e) { log.error("IOException thrown when reading entries from zip file.", e); } } } return m; } private Calendar getImportTime() { InputStream indexHtml = getFileFromZip("index.html"); if (indexHtml == null) return null; try { String s = IOUtils.toString(indexHtml); String sDate = StringUtils.substringBetween(s, "<tr><th>Date:</th><td>", "</td></tr>"); // Mon Aug 4 13:18:00 2014 return DateUtils.getCalendarFromString("EEE MMM dd HH:mm:ss yyyy", sDate); } catch (IOException e) { log.error("IOException reading inputstream index.html in getTestDate(indexHtml)", e); return null; } } @Nonnull @Override public ScanCheckResultBean checkFile() { try { zipFile = unpackZipStream(); if (zipFile == null) return new ScanCheckResultBean(ScanImportStatus.NULL_INPUT_ERROR); InputStream indexHtml = getFileFromZip("index.html"); if (indexHtml == null) return new ScanCheckResultBean(ScanImportStatus.WRONG_FORMAT_ERROR); Integer findingCount = getFindingCount(zipFile); if (findingCount == null) return new ScanCheckResultBean(ScanImportStatus.WRONG_FORMAT_ERROR); // else if (findingCount < 1) // return new ScanCheckResultBean(ScanImportStatus.EMPTY_SCAN_ERROR); testDate = getTestDate(indexHtml); ScanImportStatus scanImportStatus = checkTestDate(); return new ScanCheckResultBean(scanImportStatus, testDate); } finally { deleteZipFile(); } } private Integer getFindingCount(ZipFile zipFile) { if (zipFile.entries() == null) { return null; } int i = 0; Enumeration<? extends ZipEntry> entries = zipFile.entries(); while (entries.hasMoreElements()) { ZipEntry entry = entries.nextElement(); if (entry.getName().matches(REGEX_REPORT_FILE)) { i++; } } return i; } private Calendar getTestDate(InputStream indexHtml) { try { String s = IOUtils.toString(indexHtml); String sDate = StringUtils.substringBetween(s, "<tr><th>Date:</th><td>", "</td></tr>"); // Mon Aug 4 13:18:00 2014 return DateUtils.getCalendarFromString("EEE MMM dd HH:mm:ss yyyy", sDate); } catch (IOException e) { log.error("IOException reading inputstream index.html in getTestDate(indexHtml)", e); return null; } } }