Java tutorial
/** * Copyright (C) 2010-2017 Gordon Fraser, Andrea Arcuri and EvoSuite * contributors * * This file is part of EvoSuite. * * EvoSuite is free software: you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published * by the Free Software Foundation, either version 3.0 of the License, or * (at your option) any later version. * * EvoSuite is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with EvoSuite. If not, see <http://www.gnu.org/licenses/>. */ package org.evosuite.regression; import org.apache.commons.lang3.StringUtils; import org.evosuite.PackageInfo; import org.evosuite.runtime.mock.EvoSuiteMock; import org.evosuite.testcase.statements.MethodStatement; import org.evosuite.testcase.statements.Statement; import java.lang.reflect.Modifier; import java.net.URLClassLoader; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; import dk.brics.automaton.RegExp; public class RegressionExceptionHelper { private static final List<Class<?>> INVALID_EXCEPTIONS = Arrays .asList(new Class<?>[] { StackOverflowError.class, // Might be thrown at different places AssertionError.class } // Depends whether assertions are enabled or not ); /** * Get a simple (and unique looking) exception name (exType or exThrowingMethodCall:exType) */ public static String simpleExceptionName(RegressionTestChromosome test, Integer statementPos, Throwable ex) { if (ex == null) { return ""; } String exception = ex.getClass().getSimpleName(); if (test.getTheTest().getTestCase().hasStatement(statementPos)) { Statement exThrowingStatement = test.getTheTest().getTestCase().getStatement(statementPos); if (exThrowingStatement instanceof MethodStatement) { String exMethodcall = ((MethodStatement) exThrowingStatement).getMethod().getName(); exception = exMethodcall + ":" + exception; } } return exception; } /** * Get signature based on the root cause from the stack trace * (uses: location of the error triggering line from CUT) * @param CUT the class to expect the exception to be thrown from * @return signature string */ public static String getExceptionSignature(Throwable throwable, String CUT) { String signature = throwable.getClass().getSimpleName(); StackTraceElement[] stackTrace = throwable.getStackTrace(); for (StackTraceElement el : stackTrace) { String elClass = el.getClassName(); if (!Objects.equals(elClass, CUT)) { continue; } String method = el.getMethodName(); int line = el.getLineNumber(); signature += ":" + method + "-" + line; break; } return signature; } /** * Calculate the number of different exceptions, given two sets of exceptions. */ public static int compareExceptionDiffs(Map<Integer, Throwable> originalExceptionMapping, Map<Integer, Throwable> regressionExceptionMapping) { int exDiff = (int) Math.abs((originalExceptionMapping.size() - regressionExceptionMapping.size())); /* * If the number of exceptions is different, clearly the executions are * propagating to different results. Otherwise, we need to compare the * same assertions to make sure they're actually the same. */ if (exDiff == 0) { // For all exceptions thrown original class for (Map.Entry<Integer, Throwable> origException : originalExceptionMapping.entrySet()) { boolean skip = false; // Skip if the exception or the message are null // Sometimes the getMesage may call the CUT's exception handler which may crash try { if (origException.getValue() == null || origException.getValue().getMessage() == null) { originalExceptionMapping.remove(origException.getKey()); skip = true; } } catch (Throwable t) { continue; } // See if exception throwing classes differed try { Throwable x = origException.getValue(); Class<?> ex = getExceptionClassToUse(x); String sourceClass = getSourceClassName(x); if (sourceClass != null && isValidSource(sourceClass) && isExceptionToAssertValid(ex)) { // Get other exception throwing class and compare them Throwable otherX = regressionExceptionMapping.get(origException.getKey()); String otherSourceClass = getSourceClassName(otherX); if (!sourceClass.equals(otherSourceClass)) { exDiff++; //logger.warn("Exception throwing classes differed: {} {}", sourceClass, otherSourceClass); } } } catch (Throwable t) { // ignore } // Skip if the exceptions are not comparable try { if (regressionExceptionMapping.containsKey(origException.getKey()) && (regressionExceptionMapping.get(origException.getKey()) == null || regressionExceptionMapping.get(origException.getKey()) .getMessage() == null)) { regressionExceptionMapping.remove(origException.getKey()); skip = true; } } catch (Throwable t) { continue; } // If they start differing from @objectID skip this one if (!skip && regressionExceptionMapping.get(origException.getKey()) != null) { String origExceptionMessage = origException.getValue().getMessage(); String regExceptionMessage = regressionExceptionMapping.get(origException.getKey()) .getMessage(); int diffIndex = StringUtils.indexOfDifference(origExceptionMessage, regExceptionMessage); if (diffIndex > 0) { if (origExceptionMessage.charAt(diffIndex - 1) == '@') { originalExceptionMapping.remove(origException.getKey()); regressionExceptionMapping.remove(origException.getKey()); skip = true; } else { // If @ is in the last 10 characters, it's likely an object pointer comparison issue int howFarBack = 10; if (diffIndex > howFarBack) { String last10 = origExceptionMessage.substring(diffIndex - howFarBack, diffIndex); if (last10.contains("@")) { originalExceptionMapping.remove(origException.getKey()); regressionExceptionMapping.remove(origException.getKey()); skip = true; } } } } } // ignore security manager exceptions if (!skip && origException.getValue().getMessage().contains("Security manager blocks")) { originalExceptionMapping.remove(origException.getKey()); regressionExceptionMapping.remove(origException.getKey()); skip = true; } if (skip) { continue; } // do the comparison if (!regressionExceptionMapping.containsKey(origException.getKey()) || (!regressionExceptionMapping .get(origException.getKey()).getMessage().equals(origException.getValue().getMessage()))) { exDiff++; } } // For all exceptions in the regression class. // Any bad exceptions were removed from this object earlier for (Map.Entry<Integer, Throwable> regException : regressionExceptionMapping.entrySet()) { if (!originalExceptionMapping.containsKey(regException.getKey())) { exDiff++; } } } return exDiff; } /** * Add regression-diff comments for exception messages */ public static void addExceptionAssertionComments(RegressionTestChromosome regressionTest, Map<Integer, Throwable> originalExceptionMapping, Map<Integer, Throwable> regressionExceptionMapping) { for (Map.Entry<Integer, Throwable> original : originalExceptionMapping.entrySet()) { int originalStatementPos = original.getKey(); Throwable originalException = original.getValue(); if (!regressionExceptionMapping.containsKey(originalStatementPos)) { if (testStatementCommentNotContains(regressionTest, originalStatementPos, "modified version")) { addExceptionDifferenceComment(regressionTest, originalException, originalStatementPos, "The modified version did not exhibit this exception", false); //FIXME: for some reason we're not adding this comment to the other version! } } else { if (originalException != null && originalException.getMessage() != null) { // compare the exception messages if (!originalException.getMessage() .equals(regressionExceptionMapping.get(originalStatementPos).getMessage())) { if (testStatementCommentNotContains(regressionTest, originalStatementPos, "EXCEPTION DIFF:")) { regressionTest.getTheTest().getTestCase().getStatement(originalStatementPos).addComment( "EXCEPTION DIFF:\nDifferent Exceptions were thrown:\nOriginal Version:\n " + originalException.getClass().getName() + " : " + originalException.getMessage() + "\nModified Version:\n " + regressionExceptionMapping.get(originalStatementPos).getClass() .getName() + " : " + regressionExceptionMapping.get(originalStatementPos).getMessage() + "\n"); } } else { // Compare the classes throwing the exception Class<?> ex = getExceptionClassToUse(originalException); String sourceClass = getSourceClassName(originalException); if (sourceClass != null && isValidSource(sourceClass) && isExceptionToAssertValid(ex) && regressionExceptionMapping.get(originalStatementPos) != null) { Throwable otherX = regressionExceptionMapping.get(originalStatementPos); String otherSourceClass = getSourceClassName(otherX); if (!sourceClass.equals(otherSourceClass)) { if (testStatementCommentNotContains(regressionTest, originalStatementPos, "EXCEPTION DIFF:")) { regressionTest.getTheTest().getTestCase().getStatement(originalStatementPos) .addComment( "EXCEPTION DIFF:\nExceptions thrown by different classes:\nOriginal Version:\n " + sourceClass + "\nModified Version:\n " + otherSourceClass + "\n"); } } } } } // If both show the same error, pop the error from the // regression exception map, to get to a diff. regressionExceptionMapping.remove(originalStatementPos); } } for (Map.Entry<Integer, Throwable> regression : regressionExceptionMapping.entrySet()) { Throwable regressionException = regression.getValue(); int regressionStatementPos = regression.getKey(); if (testStatementCommentNotContains(regressionTest, regressionStatementPos, "original version")) { addExceptionDifferenceComment(regressionTest, regressionException, regressionStatementPos, "The original version did not exhibit this exception", true); } } } private static boolean testStatementCommentNotContains(RegressionTestChromosome test, int statementPos, String compareComment) { return (test.getTheTest().getTestCase().hasStatement(statementPos) && !test.getTheTest().getTestCase() .getStatement(statementPos).getComment().contains(compareComment)); } /** * Add exception difference comment to test case, at given position * @param test the test case to add the comment to * @param t throwable (Exception) which has occurred * @param statementPos the position to add the comment on * @param comment the comment string * @param addToTestForOtherClassLoader whether to add the same comment on the test on reg. CL */ private static void addExceptionDifferenceComment(RegressionTestChromosome test, Throwable t, int statementPos, String comment, boolean addToTestForOtherClassLoader) { String exceptionDiffComment = "EXCEPTION DIFF:\n" + comment + ":\n " + t.getClass().getName() + " : " + t.getMessage() + "\n\n"; test.getTheTest().getTestCase().getStatement(statementPos).addComment(exceptionDiffComment); if (addToTestForOtherClassLoader) { test.getTheSameTestForTheOtherClassLoader().getTestCase().getStatement(statementPos) .addComment(exceptionDiffComment); } } /** * This part is "temporarily" copied over from TestCodeVisitor. * Until they are made statically available to use in this class. */ private static String getSourceClassName(Throwable exception) { if (exception.getStackTrace().length == 0) { return null; } return exception.getStackTrace()[0].getClassName(); } private static boolean isValidSource(String sourceClass) { return (!sourceClass.startsWith(PackageInfo.getEvoSuitePackage() + ".") || sourceClass.startsWith(PackageInfo.getEvoSuitePackage() + ".runtime.")) && !sourceClass.equals(URLClassLoader.class.getName()) && // Classloaders may differ, e.g. when running with ant !sourceClass.startsWith(RegExp.class.getPackage().getName()) && !sourceClass.startsWith("java.lang.System") && !sourceClass.startsWith("java.lang.String") && !sourceClass.startsWith("sun.") && !sourceClass.startsWith("com.sun.") && !sourceClass.startsWith("jdk.internal."); } private static boolean isExceptionToAssertValid(Class<?> exceptionClass) { return !INVALID_EXCEPTIONS.contains(exceptionClass); } private static Class<?> getExceptionClassToUse(Throwable exception) { /* we can only catch a public class. for "readability" of tests, it shouldn't be a mock one either */ Class<?> ex = exception.getClass(); while (!Modifier.isPublic(ex.getModifiers()) || EvoSuiteMock.class.isAssignableFrom(ex) || ex.getCanonicalName().startsWith("com.sun.")) { ex = ex.getSuperclass(); } return ex; } }