Java tutorial
//////////////////////////////////////////////////////////////////////// // // Copyright (c) 2009-2015 Denim Group, Ltd. // // The contents of this file are subject to the Mozilla Public License // Version 2.0 (the "License"); you may not use this file except in // compliance with the License. You may obtain a copy of the License at // http://www.mozilla.org/MPL/ // // Software distributed under the License is distributed on an "AS IS" // basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the // License for the specific language governing rights and limitations // under the License. // // The Original Code is ThreadFix. // // The Initial Developer of the Original Code is Denim Group, Ltd. // Portions created by Denim Group, Ltd. are Copyright (C) // Denim Group, Ltd. All Rights Reserved. // // Contributor(s): Denim Group, Ltd. // //////////////////////////////////////////////////////////////////////// package com.denimgroup.threadfix.importer.impl.remoteprovider; import com.denimgroup.threadfix.CollectionUtils; import com.denimgroup.threadfix.annotations.RemoteProvider; import com.denimgroup.threadfix.data.entities.*; import com.denimgroup.threadfix.data.enums.QualysPlatform; import com.denimgroup.threadfix.importer.impl.remoteprovider.utils.DefaultRequestConfigurer; import com.denimgroup.threadfix.importer.impl.remoteprovider.utils.HttpResponse; import com.denimgroup.threadfix.importer.impl.remoteprovider.utils.RemoteProviderHttpUtils; import com.denimgroup.threadfix.importer.util.DateUtils; import com.denimgroup.threadfix.importer.util.HandlerWithBuilder; import com.denimgroup.threadfix.importer.util.ScanUtils; import org.xml.sax.Attributes; import org.xml.sax.SAXException; import org.apache.commons.lang3.StringEscapeUtils; import javax.annotation.Nonnull; import javax.xml.bind.DatatypeConverter; import java.io.InputStream; import java.util.*; import static com.denimgroup.threadfix.CollectionUtils.list; import static com.denimgroup.threadfix.CollectionUtils.map; import static com.denimgroup.threadfix.CollectionUtils.set; import static com.denimgroup.threadfix.importer.impl.remoteprovider.utils.RemoteProviderHttpUtilsImpl.getImpl; /** * TODO use POST data to pre-filter web requests * @author mcollins * */ @RemoteProvider(name = "QualysGuard WAS") public class QualysRemoteProvider extends AbstractRemoteProvider { public String getType() { return ScannerType.QUALYSGUARD_WAS.getDisplayName(); } private String username = null; private String password = null; private String platform = null; private static final Map<String, String> SEVERITIES_MAP = new HashMap<String, String>() { { put("150000", "5"); put("150001", "5"); put("150002", "5"); put("150003", "5"); put("150004", "2"); put("150005", "2"); put("150006", "1"); put("150007", "1"); put("150008", "1"); put("150009", "1"); put("150010", "1"); put("150011", "3"); put("150012", "5"); put("150013", "5"); put("150014", "1"); put("150015", "1"); put("150016", "2"); put("150017", "1"); put("150018", "2"); put("150019", "2"); put("150020", "1"); put("150021", "1"); put("150022", "3"); put("150023", "2"); put("150024", "1"); put("150025", "1"); put("150026", "1"); put("150028", "1"); put("150029", "1"); put("150030", "1"); put("150032", "3"); put("150033", "3"); put("150034", "3"); put("150035", "1"); put("150036", "1"); put("150037", "1"); put("150038", "1"); put("150039", "1"); put("150040", "1"); put("150041", "1"); put("150042", "3"); put("150043", "1"); put("150044", "3"); put("150045", "3"); put("150046", "5"); put("150047", "5"); put("150048", "5"); put("150049", "4"); put("150051", "3"); put("150052", "5"); put("150053", "3"); put("150054", "1"); put("150055", "5"); put("150056", "2"); put("150057", "5"); put("150058", "1"); put("150059", "1"); put("150060", "5"); put("150061", "1"); put("150062", "5"); put("150063", "1"); put("150064", "1"); put("150065", "2"); put("150066", "1"); put("150067", "3"); put("150068", "1"); put("150069", "4"); put("150071", "3"); put("150072", "2"); put("150076", "4"); put("150077", "1"); put("150078", "1"); put("150079", "3"); put("150080", "3"); put("150081", "1"); put("150082", "1"); put("150083", "1"); put("150084", "1"); put("150085", "3"); put("150086", "3"); put("150087", "1"); put("150088", "1"); put("150089", "3"); put("150090", "5"); put("150092", "5"); put("150093", "5"); put("150094", "1"); put("150095", "1"); put("150097", "1"); put("150098", "2"); put("150099", "1"); put("150100", "1"); put("150101", "1"); put("150103", "1"); put("150104", "1"); put("150105", "1"); put("150106", "1"); put("150107", "1"); put("150108", "1"); put("150109", "1"); put("150111", "1"); put("150112", "2"); put("150114", "3"); put("150115", "1"); put("150116", "1"); put("150118", "1"); put("150120", "3"); put("150121", "3"); put("150122", "2"); put("150123", "2"); put("150124", "3"); put("150125", "1"); put("150127", "5"); put("150128", "1"); put("150129", "2"); put("150134", "5"); } }; RemoteProviderHttpUtils utils = getImpl(this.getClass()); public QualysRemoteProvider() { super(ScannerType.QUALYSGUARD_WAS); } private enum QualysScanDetailParam { ACTION("action"), DETAILS("details"), IDS("ids"); private String param; QualysScanDetailParam(String param) { this.param = param; } public String getParam() { return this.param; } } @Override public List<Scan> getScans(RemoteProviderApplication remoteProviderApplication) { if (remoteProviderApplication == null || remoteProviderApplication.getRemoteProviderType() == null) { LOG.error("Null input to Qualys getScan(), returning null."); return null; } password = remoteProviderApplication.getRemoteProviderType().getPassword(); username = remoteProviderApplication.getRemoteProviderType().getUsername(); List<String> scanIds = mostRecentScanForApp(remoteProviderApplication); if (scanIds == null || scanIds.size() == 0) { LOG.warn("No valid scans were found."); return null; } List<Scan> scanList = list(); for (String scanId : scanIds) { HttpResponse response = utils.getUrl( getScanUrl(remoteProviderApplication.getRemoteProviderType()) + scanId, username, password); if (response.isValid()) { inputStream = response.getInputStream(); } else { LOG.warn("Got a " + response.getStatus() + " response code when requesting scan with ID " + scanId + ". Trying the next scan."); continue; } QualysWASSAXParser scanParser = new QualysWASSAXParser(); Scan resultScan = parseSAXInput(scanParser); Set<String> qidSet = set(); for (Finding finding : resultScan) { qidSet.add(finding.getChannelVulnerability().getCode()); } String qids = CollectionUtils.join(",", qidSet); String[] parameters = { QualysScanDetailParam.ACTION.getParam(), QualysScanDetailParam.IDS.getParam(), QualysScanDetailParam.DETAILS.getParam() }; String[] values = { "list", qids, "All" }; String[] headerNames = { "X-Requested-With", "Content-Type" }; String[] headerVals = { "Curl", "application/x-www-form-urlencoded" }; response = utils.postUrl(getScanDetailsUrl(remoteProviderApplication.getRemoteProviderType()), parameters, values, username, password, headerNames, headerVals); if (response.isValid()) { inputStream = response.getInputStream(); parseQualysSAXInput(); LOG.info("Retrieved additional scanner details for QID: " + qids); } else { LOG.warn("Unable to retrieve scan details for the application " + remoteProviderApplication.getNativeName() + ". Got response code " + response.getStatus()); } LOG.info("The Qualys scan import for scan ID " + scanId + " was successful."); resultScan.setApplicationChannel(remoteProviderApplication.getApplicationChannel()); scanList.add(resultScan); } return scanList; } @Override public List<RemoteProviderApplication> fetchApplications() { if (remoteProviderType == null || remoteProviderType.getUsername() == null || remoteProviderType.getPassword() == null) { LOG.error("Insufficient credentials given to Qualys fetchApplications()."); return null; } LOG.info("Fetching Qualys applications."); password = remoteProviderType.getPassword(); username = remoteProviderType.getUsername(); // POST with no parameters // TODO include filters String appsUrl = getAppsUrl(remoteProviderType); boolean keepGoing = true; long currentId = 0L; List<RemoteProviderApplication> applications = list(); int tooManyTimes = 200; int currentTimes = 0; while (keepGoing) { HttpResponse connection = utils.postUrlWithConfigurer(appsUrl, getRequestConfigurer(1000, currentId)); InputStream stream; if (connection.isValid()) { stream = connection.getInputStream(); } else { LOG.warn("Failed to retrieve the applications. Check your credentials. status code was " + connection.getStatus()); return null; } QualysAppsParser parser = new QualysAppsParser(); parse(stream, parser); if (parser.list.size() > 0) { LOG.info("Number of Qualys applications: " + parser.list.size()); applications.addAll(parser.list); } else { LOG.warn("No Qualys applications were found. Check your configuration."); } keepGoing = parser.lastId != currentId && ++currentTimes < tooManyTimes && parser.hasMoreRecords; currentId = parser.lastId; } return applications; } private DefaultRequestConfigurer getRequestConfigurer(int numResults, long lastId) { String contents = "<ServiceRequest>\n" + " <preferences>\n" + " <limitResults>" + numResults + "</limitResults>\n" + " </preferences>\n"; if (lastId > 0) { contents = contents + " <filters>\n" + " <Criteria field=\"id\" operator=\"GREATER\">" + lastId + "</Criteria>\n" + " </filters>\n"; } contents = contents + "</ServiceRequest>"; return new DefaultRequestConfigurer().withUsernamePassword(username, password).withRequestBody(contents, null); } private DefaultRequestConfigurer getScanFilterRequestConfigurer(int numResults, String appName) { String contents = "<ServiceRequest>\n" + " <preferences>\n" + " <limitResults>" + numResults + "</limitResults>\n" + " </preferences>\n" + " <filters>\n" + " <Criteria field=\"webApp.name\" operator=\"CONTAINS\">" + StringEscapeUtils.escapeXml10(appName) + "</Criteria>\n" + " <Criteria field=\"status\" operator=\"EQUALS\">FINISHED</Criteria>\n" + " </filters>\n" + "</ServiceRequest>"; return new DefaultRequestConfigurer().withUsernamePassword(username, password).withRequestBody(contents, null); } public List<String> mostRecentScanForApp(RemoteProviderApplication app) { if (app == null || app.getNativeName() == null) { return null; } HttpResponse response = utils.postUrlWithConfigurer(getScansForAppUrl(app.getRemoteProviderType()), getScanFilterRequestConfigurer(1000, app.getNativeName())); InputStream stream; if (response.isValid()) { stream = response.getInputStream(); } else { LOG.warn("Unable to retrieve scans for the application " + app.getNativeName() + ". Got response code " + response.getStatus()); return null; } QualysScansForAppParser parser = new QualysScansForAppParser(); parse(stream, parser); List<String> scanIds = list(); // This should be replaced with the filtered code for (Map<String, String> map : parser.list) { if (app.getNativeName().equals(map.get("webAppName")) && map.get("date") != null) { Calendar mapDate = DateUtils.getCalendarFromUTCString(map.get("date")); if (mapDate != null && (app.getLastImportTime() == null || mapDate.after(app.getLastImportTime()))) { scanIds.add(map.get("id")); } } } LOG.info("Returning scan IDs " + scanIds + " for application " + app.getNativeName()); return scanIds; } private static String getBaseUrl(RemoteProviderType type) { QualysPlatform qp; String platform = type.getPlatform(); if (platform == null || platform.isEmpty()) { qp = type.getIsEuropean() ? QualysPlatform.EU : QualysPlatform.US_1; } else { qp = QualysPlatform.getPlatform(platform); } return qp.getUrl(); } public static String getScansForAppUrl(RemoteProviderType type) { return getBaseUrl(type) + "/qps/rest/3.0/search/was/wasscan"; } public static String getScanUrl(RemoteProviderType type) { return getBaseUrl(type) + "/qps/rest/3.0/download/was/wasscan/"; } public static String getAppsUrl(RemoteProviderType type) { return getBaseUrl(type) + "/qps/rest/3.0/search/was/webapp"; } public static String getScanDetailsUrl(RemoteProviderType type) { return getBaseUrl(type) + "/api/2.0/fo/knowledge_base/vuln/"; } // PARSE FUNCTION @Nonnull private void parseQualysSAXInput() { log.debug("Starting Qualys SAX Parsing."); if (inputStream == null) { throw new IllegalStateException( "InputStream was null. Can't parse SAX input. This is probably a coding error."); } // we don't need the state from the details parser so we can just pass the new object in ScanUtils.readSAXInput(new QualysWASSAXDetailsParser(), "Done Parsing.", inputStream); if (shouldDeleteAfterParsing) { deleteScanFile(); } } // PARSER CLASSES class QualysAppsParser extends HandlerWithBuilder { @Nonnull public List<RemoteProviderApplication> list = list(); public boolean hasMoreRecords; public long lastId = 0L; private boolean getName = false, getHasMoreRecords = false, getLastId = false; public void startElement(String uri, String name, String qName, Attributes atts) throws SAXException { if (qName.equals("name")) { getName = true; } else if (qName.equals("hasMoreRecords")) { getHasMoreRecords = true; } else if (qName.equals("lastId")) { getLastId = true; } } public void endElement(String uri, String name, String qName) { if (getName) { String tempNameString = getBuilderText(); RemoteProviderApplication remoteProviderApplication = new RemoteProviderApplication(); remoteProviderApplication.setNativeName(tempNameString); remoteProviderApplication.setRemoteProviderType(remoteProviderType); list.add(remoteProviderApplication); getName = false; } else if (getHasMoreRecords) { String text = getBuilderText(); hasMoreRecords = "true".equals(text); getHasMoreRecords = false; } else if (getLastId) { String text = getBuilderText(); lastId = text != null && text.matches("[0-9]+") ? Long.valueOf(text) : 0L; getLastId = false; } } public void characters(char ch[], int start, int length) { if (getName || getHasMoreRecords || getLastId) { addTextToBuilder(ch, start, length); } } } private class QualysScansForAppParser extends HandlerWithBuilder { public List<Map<String, String>> list = list(); private boolean inWebApp = false; private boolean getName = false; private String webAppName = ""; private boolean getId = false; private String currentId = null; private boolean getStatus = false; private String currentStatus = null; private boolean getDate = false; private String currentDate = null; public void startElement(String uri, String name, String qName, Attributes atts) throws SAXException { if (qName.equals("webApp")) { inWebApp = true; } else if (currentId == null && qName.equals("id")) { getId = true; } else if (inWebApp && qName.equals("name")) { getName = true; inWebApp = false; } else if (qName.equals("status")) { getStatus = true; } else if (qName.equals("launchedDate")) { getDate = true; } } @Override public void endElement(String uri, String localName, String qName) throws SAXException { if (getId) { currentId = getBuilderText(); getId = false; } else if (getStatus) { currentStatus = getBuilderText(); getStatus = false; } else if (getDate) { currentDate = getBuilderText(); getDate = false; } else if (getName) { webAppName = getBuilderText(); getName = false; } if (qName.equals("WasScan")) { Map<String, String> myMap = map("id", currentId, "status", currentStatus, "date", currentDate, "webAppName", webAppName); list.add(myMap); currentStatus = null; currentId = null; currentDate = null; webAppName = null; } } public void characters(char ch[], int start, int length) { if (getId || getStatus || getDate || getName) { addTextToBuilder(ch, start, length); } } } private class QualysWASSAXParser extends HandlerWithBuilder { private Boolean getDate = false; private Boolean getUri = false; private Boolean getParameter = false; private Boolean getChannelVulnName = false; private Boolean getAttackDetail = false; private boolean getAttackResponse = false; private boolean isBase64 = false; private String currentChannelVulnCode = null; private String currentPath = null; private String currentParameter = null; private String currentSeverityCode = null; private String currentAttackDetail = null; private String currentAttackResponse = null; private Map<FindingKey, String> findingMap = map(); public void add(Finding finding) { if (finding != null) { finding.setNativeId(getNativeId(finding)); finding.setIsStatic(false); saxFindingList.add(finding); } } //////////////////////////////////////////////////////////////////// // Event handlers. //////////////////////////////////////////////////////////////////// public void startElement(String uri, String name, String qName, Attributes atts) { if (qName.equals("launchedDate")) { getDate = true; } else if (qName.equals("uri")) { getUri = true; } else if (qName.equals("qid")) { getChannelVulnName = true; } else if (qName.equals("param")) { getParameter = true; } else if (qName.equals("payload")) { getAttackDetail = true; } else if (qName.equals("result")) { getAttackResponse = true; isBase64 = "true".equals(atts.getValue("base64")); } else if (qName.equals("instances")) { currentSeverityCode = SEVERITIES_MAP.get(currentChannelVulnCode); if (currentSeverityCode == null) { LOG.warn("Unable to retrieve severity for code " + currentChannelVulnCode + ". Setting to 3"); currentSeverityCode = "3"; } findingMap.put(FindingKey.PATH, currentPath); findingMap.put(FindingKey.PARAMETER, currentParameter); findingMap.put(FindingKey.VULN_CODE, currentChannelVulnCode); findingMap.put(FindingKey.SEVERITY_CODE, currentSeverityCode); findingMap.put(FindingKey.VALUE, currentAttackDetail); findingMap.put(FindingKey.ATTACK_STRING, currentAttackDetail); findingMap.put(FindingKey.REQUEST, currentAttackDetail); findingMap.put(FindingKey.RESPONSE, currentAttackResponse); Finding finding = constructFinding(findingMap); add(finding); currentParameter = null; currentPath = null; getParameter = false; } } public void endElement(String uri, String name, String qName) { if (getDate) { String tempDateString = getBuilderText(); if (tempDateString != null && !tempDateString.trim().isEmpty()) { date = DateUtils.getCalendarFromUTCString(tempDateString); } getDate = false; } else if (getUri) { currentPath = getBuilderText(); getUri = false; } else if (getChannelVulnName) { currentChannelVulnCode = getBuilderText(); getChannelVulnName = false; } else if (getParameter) { currentParameter = getBuilderText(); getParameter = false; } else if (getAttackDetail) { currentAttackDetail = getBuilderText(); getAttackDetail = false; } else if (getAttackResponse) { currentAttackResponse = getBuilderText(); if (isBase64) { currentAttackResponse = currentAttackResponse.replace('_', '/').replace('-', '+'); currentAttackResponse = new String(DatatypeConverter.parseBase64Binary(currentAttackResponse)); } getAttackResponse = false; } } public void characters(char ch[], int start, int length) { if (getAttackResponse || getDate || getUri || getChannelVulnName || getParameter || getAttackDetail) { addTextToBuilder(ch, start, length); } } } private class QualysWASSAXDetailsParser extends HandlerWithBuilder { private Boolean getDiagnosis = false; private Boolean getConsequence = false; private Boolean getSolution = false; private Boolean getQid = false; private String currentQid = null; private String currentDiagnosis = null; private String currentConsequence = null; private String currentSolution = null; private Map<FindingKey, String> findingMap = map(); //////////////////////////////////////////////////////////////////// // Event handlers. //////////////////////////////////////////////////////////////////// public void startElement(String uri, String name, String qName, Attributes atts) throws SAXException { if (qName.equalsIgnoreCase("diagnosis")) { getDiagnosis = true; } else if (qName.equalsIgnoreCase("consequence")) { getConsequence = true; } else if (qName.equalsIgnoreCase("qid")) { getQid = true; } else if (qName.equalsIgnoreCase("solution")) { getSolution = true; } else if (qName.equalsIgnoreCase("discovery")) { String currentDetail = currentDiagnosis + currentConsequence; findingMap.put(FindingKey.RECOMMENDATION, currentSolution); findingMap.put(FindingKey.DETAIL, currentDetail); for (Finding finding : saxFindingList) { if (finding.getChannelVulnerability().getCode().equals(currentQid) && finding.getScannerDetail() == null && finding.getScannerRecommendation() == null) { findingMap.put(FindingKey.VALUE, finding.getAttackString()); addFindingDetail(finding, findingMap); } } } } public void endElement(String uri, String name, String qName) { if (getDiagnosis) { currentDiagnosis = getBuilderText(); getDiagnosis = false; } else if (getConsequence) { currentConsequence = getBuilderText(); getConsequence = false; } else if (getQid) { currentQid = getBuilderText(); getQid = false; } else if (getSolution) { currentSolution = getBuilderText(); getSolution = false; } } public void characters(char ch[], int start, int length) { if (getDiagnosis || getConsequence || getSolution || getQid) { addTextToBuilder(ch, start, length); } } } }