edu.umd.cs.submitServer.servlets.ReportTestOutcomes.java Source code

Java tutorial

Introduction

Here is the source code for edu.umd.cs.submitServer.servlets.ReportTestOutcomes.java

Source

/**
 * Marmoset: a student project snapshot, submission, testing and code review
 * system developed by the Univ. of Maryland, College Park
 * 
 * Developed as part of Jaime Spacco's Ph.D. thesis work, continuing effort led
 * by William Pugh. See http://marmoset.cs.umd.edu/
 * 
 * Copyright 2005 - 2011, Univ. of Maryland
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 * 
 */

/*
 * Created on Jan 16, 2005
 *
 * @author jspacco
 */
package edu.umd.cs.submitServer.servlets;

import java.io.IOException;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;

import javax.mail.MessagingException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.apache.commons.fileupload.FileItem;
import org.apache.log4j.Logger;

import edu.umd.cs.marmoset.modelClasses.BuildServer;
import edu.umd.cs.marmoset.modelClasses.CodeMetrics;
import edu.umd.cs.marmoset.modelClasses.Project;
import edu.umd.cs.marmoset.modelClasses.ServerError;
import edu.umd.cs.marmoset.modelClasses.Student;
import edu.umd.cs.marmoset.modelClasses.StudentRegistration;
import edu.umd.cs.marmoset.modelClasses.StudentSubmitStatus;
import edu.umd.cs.marmoset.modelClasses.Submission;
import edu.umd.cs.marmoset.modelClasses.TestOutcome;
import edu.umd.cs.marmoset.modelClasses.TestOutcomeCollection;
import edu.umd.cs.marmoset.modelClasses.TestRun;
import edu.umd.cs.marmoset.modelClasses.TestRun.Kind;
import edu.umd.cs.marmoset.modelClasses.TestSetup;
import edu.umd.cs.marmoset.utilities.JavaMail;
import edu.umd.cs.submitServer.InvalidRequiredParameterException;
import edu.umd.cs.submitServer.MultipartRequest;
import edu.umd.cs.submitServer.WebConfigProperties;
import edu.umd.cs.submitServer.filters.SubmitServerFilter;

/**
 * @author jspacco Called (usually by the Build Server) to report the outcomes
 *         of a test run
 */
public class ReportTestOutcomes extends SubmitServerServlet {
    private static final WebConfigProperties webProperties = WebConfigProperties.get();
    private static Logger failedLog;

    private static Logger getFailedBackgroundRetestLog() {
        if (failedLog == null) {
            failedLog = Logger.getLogger("edu.umd.cs.submitServer.servlets.failedBackgroundRetestLog");
        }
        return failedLog;
    }

    private static Logger successfulLog;

    private static Logger getSuccessfulBackgroundRetestLog() {
        if (successfulLog == null) {
            successfulLog = Logger.getLogger("edu.umd.cs.submitServer.servlets.successfulBackgroundRetestLog");
        }
        return successfulLog;
    }

    @Override
    public void doPost(HttpServletRequest request, HttpServletResponse response)
            throws ServletException, IOException {
        // will be set by MultipartRequestFilter
        MultipartRequest multipartRequest = (MultipartRequest) request.getAttribute(MULTIPART_REQUEST);

        Timestamp now = new Timestamp(System.currentTimeMillis());

        // Insert test outcomes into database,
        // create new TestRun
        // and update submission as having been tested.
        Connection conn = null;
        boolean transactionSuccess = false;
        try {
            // Get submission pk and the submission
            @Submission.PK
            int submissionPK = Submission.asPK(multipartRequest.getIntParameter("submissionPK"));

            String courses = multipartRequest.getStringParameter("courses");

            // Get the testSetupPK
            int testSetupPK = multipartRequest.getIntParameter("testSetupPK");
            boolean newTestSetup = multipartRequest.getBooleanParameter("newTestSetup");
            boolean isBackgroundRetest = multipartRequest.getBooleanParameter("isBackgroundRetest");
            String load = multipartRequest.getOptionalStringParameter("load");

            if (load == null)
                load = "unknown";
            String kindString = multipartRequest.getOptionalStringParameter("kind");
            Kind kind = Kind.UNKNOWN;
            if (kindString != null) {
                try {
                    kind = Kind.valueOf(kindString);
                } catch (RuntimeException e) {
                    // ignore
                }

            }

            String remoteHost = SubmitServerFilter.getRemoteHost(request);
            // Get test machine (if specified)
            String testMachine = multipartRequest.getOptionalStringParameter("hostname");
            if (testMachine == null)
                testMachine = multipartRequest.getOptionalStringParameter("testMachine");
            if (testMachine == null)
                testMachine = "unknown";

            CodeMetrics codeMetrics = getCodeMetrics(multipartRequest);
            int testDurationsMillis = multipartRequest.getIntegerParameter("testDurationsMillis", 0);

            // Read into TestOutcomeCollection in memory
            TestOutcomeCollection testOutcomeCollection = TestOutcomeCollection
                    .deserialize(getfileItemDataAndDelete(multipartRequest));

            logHotspotErrors(submissionPK, testSetupPK, testMachine, testOutcomeCollection);

            conn = getConnection();

            Submission submission = Submission.lookupBySubmissionPK(submissionPK, conn);

            if (submission == null) {
                throw new ServletException(
                        "submissionPK " + submissionPK + " does not refer to a submission in the database");
            }

            StudentRegistration studentRegistration = StudentRegistration
                    .lookupByStudentRegistrationPK(submission.getStudentRegistrationPK(), conn);
            @Student.PK
            Integer studentPK = studentRegistration.getStudentPK();
            Project project = Project.getByProjectPK(submission.getProjectPK(), conn);

            TestSetup testSetup = TestSetup.lookupByTestSetupPK(testSetupPK, conn);
            Integer canonicalTestRunPK = testSetup.getTestRunPK();
            if (newTestSetup && (testSetup.getStatus() == TestSetup.Status.TESTED
                    || testSetup.getStatus() == TestSetup.Status.ACTIVE)) {
                newTestSetup = false;
            }
            BuildServer.insertOrUpdateSuccess(conn, testMachine, remoteHost, now, load, submission);

            // Validate buildserver
            Collection<Integer> allowedCourses = RequestSubmission.getCourses(conn, courses);

            if (allowedCourses.isEmpty()) {
                ServerError.insert(conn, ServerError.Kind.BAD_AUTHENTICATION, studentPK, studentPK,
                        project.getCoursePK(), project.getProjectPK(), submission.getSubmissionPK(), "",
                        "Build server " + testMachine
                                + " reporting outcome but does not provide any valid credentials",
                        "", this.getClass().getSimpleName(), "", "", remoteHost, "", "", null);
                response.sendError(HttpServletResponse.SC_UNAUTHORIZED, "Not authorized");
                return;
            }

            if (!allowedCourses.contains(project.getCoursePK())) {
                ServerError.insert(conn, ServerError.Kind.BAD_AUTHENTICATION, studentPK, studentPK,
                        project.getCoursePK(), project.getProjectPK(), submission.getSubmissionPK(), "",
                        "Build server " + testMachine
                                + " reporting outcome for course it is not authorized to do so",
                        "", this.getClass().getSimpleName(), "", "", remoteHost, "", "", null);
                response.sendError(HttpServletResponse.SC_UNAUTHORIZED, "Not authorized");
                return;
            }

            TestRun canonicalTestRun = TestRun.lookupByTestRunPK(canonicalTestRunPK, conn);
            if (!newTestSetup) {
                if (canonicalTestRun == null || canonicalTestRun.getTestSetupPK() != testSetupPK) {
                    ServerError.insert(conn, ServerError.Kind.UNKNOWN, studentPK, studentPK, project.getCoursePK(),
                            project.getProjectPK(), submission.getSubmissionPK(), "",
                            "Discarding stale build server result", "", this.getClass().getSimpleName(), "", "",
                            remoteHost, "", "", null);
                    response.sendError(HttpServletResponse.SC_NOT_ACCEPTABLE, "not current test setup");
                    return;
                }

                // Set point totals

                TestOutcomeCollection canonicalTestOutcomeCollection = TestOutcomeCollection
                        .lookupByTestRunPK(canonicalTestRun.getTestRunPK(), conn);
                Map<String, TestOutcome> canonicalTestOutcomeMap = new HashMap<String, TestOutcome>();
                for (TestOutcome testOutcome : canonicalTestOutcomeCollection.getAllOutcomes()) {
                    if (testOutcome.isCardinalTestType())
                        canonicalTestOutcomeMap.put(testOutcome.getTestName(), testOutcome);
                }

                for (TestOutcome testOutcome : testOutcomeCollection.getAllOutcomes())
                    if (testOutcome.isCardinalTestType()
                            && !testOutcome.getOutcome().equals(TestOutcome.COULD_NOT_RUN)) {
                        TestOutcome canonicalTestOutcome = canonicalTestOutcomeMap.get(testOutcome.getTestName());
                        if (canonicalTestOutcome == null
                                || !canonicalTestOutcome.getTestType().equals(testOutcome.getTestType())) {
                            String message = "Did not find matching canonical test outcome for "
                                    + testOutcome.getTestName() + " with outcome " + testOutcome.getOutcome();
                            ServerError.insert(conn, ServerError.Kind.UNKNOWN, studentPK, studentPK,
                                    project.getCoursePK(), project.getProjectPK(), submission.getSubmissionPK(), "",
                                    message, "", this.getClass().getSimpleName(), "", "", remoteHost, "", "", null);
                            response.sendError(HttpServletResponse.SC_NOT_ACCEPTABLE, message);
                            return;
                        }

                        testOutcome.setPointValue(canonicalTestOutcome.getPointValue());

                    }
            } else {

                // new test setup

                // set all point values to 1
                for (TestOutcome testOutcome : testOutcomeCollection) {
                    testOutcome.setPointValue(1);
                }
            }

            getSubmitServerServletLog().info("Reporting test outcome for submissionPK = " + submissionPK
                    + ", testSetupPK = " + testSetupPK + " performed by " + testMachine + ", kind = " + kind);

            // Background Retests:
            // If this was a background re-test, then we need to take special
            // steps.
            // This was a background-retest if:
            // 1) The BuildServer says that this was a background retest
            // 2) or the status has been left as "complete" or has been
            // explicitly marked "background"
            if (isBackgroundRetest
                    || !newTestSetup && submission.getBuildStatus().equals(Submission.BuildStatus.COMPLETE)) {

                // Look up current testOutcomeCollection
                TestOutcomeCollection currentTestOutcomeCollection = TestOutcomeCollection
                        .lookupByTestRunPK(submission.getCurrentTestRunPK(), conn);

                // Look up the testRun for the current TestOutcomeCollection
                TestRun currentTestRun = TestRun.lookupByTestRunPK(submission.getCurrentTestRunPK(), conn);
                if (currentTestRun == null)
                    getSubmitServerServletLog().warn("Unable to find test run " + submission.getCurrentTestRunPK());
                if (currentTestRun == null || currentTestRun.getTestSetupPK() != testSetupPK) {
                    // Retest was against a different jarfile than the current
                    // one;
                    // for now just ignore this run.

                    return;
                }

                // Compare the cardinal test outcomes from the two
                // outcomeCollections
                String differences = compareCardinalOutcomes(currentTestOutcomeCollection, testOutcomeCollection,
                        submissionPK, testSetupPK, testMachine);

                if (differences == null) {
                    // If the results are the same, great! We have more
                    // confidence that this result is correct.
                    submission.incrementNumSuccessfulBackgroundRetests();

                    getSuccessfulBackgroundRetestLog().info("Corroborating run for submissionPK = " + submissionPK
                            + ", testSetupPK = " + testSetupPK + " performed by " + testMachine);
                } else if (differences.equals("skip")) {
                    // There may have been differences but we don't care
                    // We don't re-test "could_not_run" results
                    return;
                } else {
                    // If the results differ, log which test cases were
                    // different
                    submission.incrementNumFailedBackgroundRetests();
                    // TODO: RSS feed
                    getFailedBackgroundRetestLog().warn(differences);
                    // XXX Should I insert differing outcomes into the database
                    // as re-tests?
                    if (currentTestOutcomeCollection.getValuePassedOverall() < testOutcomeCollection
                            .getValuePassedOverall())
                        isBackgroundRetest = false;

                }
                submission.update(conn);

                // If there were no differences, commit what we've done and
                // exit.
                if (differences == null)
                    return;
            } // isBackgroundRetest

            conn = switchToTransaction(conn);

            // * Create new TestRun row
            // * increment numTestOutcomes in submissions table
            // * set currentTestRunPK in submissions table
            // * set testRunPK in all the testOutcomes
            // * write the testoutcomes to the disk

            // Create new TestRun row.
            TestRun testRun = createTestRun(submissionPK, testSetupPK, testMachine, testOutcomeCollection,
                    codeMetrics, now, testDurationsMillis);

            // perform insert
            testRun.insert(conn);
            BuildServer.updateLastTestRun(conn, testMachine, testRun);

            // Insert a new codeMetrics row if we have codeMetrics data to
            // insert
            // codeMetrics data is keyed to the testRunPK
            if (codeMetrics.getCodeSegmentSize() > 0) {
                codeMetrics.setTestRunPK(testRun.getTestRunPK());
                codeMetrics.insert(conn);
            }

            // update the testRunPK of the testOutcomes we've been sent from the
            // BuildServer
            // with the testRunPK of the row we just inserted
            testOutcomeCollection.updateTestRunPK(testRun.getTestRunPK());

            // increment the number of test outcomes
            submission.setNumTestRuns(submission.getNumTestRuns() + 1);

            // Next, insert the test outcomes
            testOutcomeCollection.insert(conn);

            // if this was a new project jarfile being tested against the
            // canonical account
            if (newTestSetup) {
                getSubmitServerServletLog().info("Ran new test setup, submissionPK = " + submissionPK
                        + ", testSetupPK = " + testSetupPK + " performed by " + testMachine + ", kind = " + kind);

                // lookup the pending project

                testSetup.setValueTotalTests(testOutcomeCollection.getValuePassedOverall());
                testSetup.setValuePublicTests(testOutcomeCollection.getValuePublicTests());
                testSetup.setValueReleaseTests(testOutcomeCollection.getValueReleaseTests());
                testSetup.setValueSecretTests(testOutcomeCollection.getValueSecretTests());

                testSetup.setTestRunPK(testRun.getTestRunPK());

                if (!testOutcomeCollection.isCompileSuccessful() || testOutcomeCollection.getNumFailedOverall() > 0
                        || testOutcomeCollection.getCouldNotRunAnyCardinalTests()) {
                    // If any tests have failed, then set status to failed
                    testSetup.setStatus(TestSetup.Status.FAILED);
                } else {
                    testSetup.setStatus(TestSetup.Status.TESTED);
                }
                getSubmitServerServletLog().info("Ran new test setup, submissionPK = " + submissionPK
                        + ", testSetupPK = " + testSetupPK + " performed by " + testMachine + ", kind = " + kind
                        + ", status = " + testSetup.getStatus());

                // update pending testSetup to reflect the changes just
                // made
                testSetup.update(conn);
            }

            // Only change information about the current test_run if this
            // run used the most recent testSetup
            if (!isBackgroundRetest && (newTestSetup || testRun.getTestSetupPK() == project.getTestSetupPK())) {
                // update the pass/fail/warning stats
                submission.setCurrentTestRunPK(testRun.getTestRunPK());

                // Ensure that submission is not release eligible if the
                // student_submit_status.can_release_test is false
                StudentSubmitStatus submitStatus = StudentSubmitStatus.lookupByStudentRegistrationPKAndProjectPK(
                        submission.getStudentRegistrationPK(), submission.getProjectPK(), conn);
                boolean canReleaseTest = (submitStatus == null) || submitStatus.getCanReleaseTest();
                submission.setReleaseEligible(testOutcomeCollection.isReleaseEligible() && canReleaseTest);

                // if project's release policy is anytime, then make it release
                // eligible
                if (Project.ANYTIME.equals(project.getReleasePolicy()))
                    submission.setReleaseEligible(canReleaseTest);

                submission.setValuePassedOverall(testOutcomeCollection.getValuePassedOverall());
                submission.setCompileSuccessful(testOutcomeCollection.isCompileSuccessful());
                submission.setValuePublicTestsPassed(testOutcomeCollection.getValuePublicTestsPassed());
                submission.setValueReleaseTestsPassed(testOutcomeCollection.getValueReleaseTestsPassed());
                submission.setValueSecretTestsPassed(testOutcomeCollection.getValueSecretTestsPassed());
                submission.setNumFindBugsWarnings(testOutcomeCollection.getNumFindBugsWarnings());

                // If we're re-setting the currentTestRunPK, then find any
                // existing
                // backgroundRetests and clear them. Background re-tests are
                // always compared
                // to the current set of testOutcomes.
                submission.setNumFailedBackgroundRetests(0);
                submission.setNumSuccessfulBackgroundRetests(0);
                submission.setNumPendingBuildRequests(0);

            }
            // perform update
            // Update the status of the submission
            submission.setBuildStatus(Submission.BuildStatus.COMPLETE);
            submission.update(conn);
            getSubmitServerServletLog().info("Completed testing submissionPK = " + submissionPK + ", testSetupPK = "
                    + testSetupPK + " performed by " + testMachine + ", kind = " + kind + ", status = "
                    + testSetup.getStatus());

            conn.commit();
            transactionSuccess = true;
        } catch (InvalidRequiredParameterException e) {
            throw new ServletException(e);
        } catch (SQLException e) {
            throw new ServletException(e);
        } finally {
            rollbackIfUnsuccessfulAndAlwaysReleaseConnection(transactionSuccess, request, conn);
        }
    }

    /**
     * @param multipartRequest
     * @return
     */
    private byte[] getfileItemDataAndDelete(MultipartRequest multipartRequest) {
        // Get the fileItem
        FileItem fileItem = multipartRequest.getFileItem();
        byte[] data = fileItem.get();
        fileItem.delete();
        return data;
    }

    /**
     * @param multipartRequest
     * @return
     * @throws InvalidRequiredParameterException
     */
    private CodeMetrics getCodeMetrics(MultipartRequest multipartRequest) throws InvalidRequiredParameterException {
        CodeMetrics codeMetrics = new CodeMetrics();
        codeMetrics.setMd5sumSourcefiles(multipartRequest.getOptionalStringParameter("md5sumClassfiles"));
        codeMetrics.setMd5sumClassfiles(multipartRequest.getOptionalStringParameter("md5sumSourcefiles"));
        if (multipartRequest.hasKey("codeSegmentSize")) {
            codeMetrics.setCodeSegmentSize(multipartRequest.getIntParameter("codeSegmentSize"));
        }
        return codeMetrics;
    }

    /**
     * @param submissionPK
     * @param testSetupPK
     * @param testMachine
     * @param testOutcomeCollection
     * @param codeMetrics
     * @param now
     * @param testDurationsMillis
     * @return
     */
    private TestRun createTestRun(@Submission.PK int submissionPK, int testSetupPK, String testMachine,
            TestOutcomeCollection testOutcomeCollection, CodeMetrics codeMetrics, Timestamp now,
            int testDurationsMillis) {
        TestRun testRun = new TestRun();
        testRun.setSubmissionPK(submissionPK);
        testRun.setTestSetupPK(testSetupPK);
        testRun.setTestMachine(testMachine);
        testRun.setTestTimestamp(now);
        testRun.setValuePassedOverall(testOutcomeCollection.getValuePassedOverall());
        testRun.setCompileSuccessful(testOutcomeCollection.isCompileSuccessful());
        testRun.setValuePublicTestsPassed(testOutcomeCollection.getValuePublicTestsPassed());
        testRun.setValueReleaseTestsPassed(testOutcomeCollection.getValueReleaseTestsPassed());
        testRun.setValueSecretTestsPassed(testOutcomeCollection.getValueSecretTestsPassed());
        testRun.setNumFindBugsWarnings(testOutcomeCollection.getNumFindBugsWarnings());
        // set the md5sum for this testRun
        // XXX currently the md5sums are stored in both testRuns and
        // codeMetrics tables
        if (codeMetrics.getMd5sumClassfiles() != null && codeMetrics.getMd5sumSourcefiles() != null) {
            testRun.setMd5sumClassfiles(codeMetrics.getMd5sumClassfiles());
            testRun.setMd5sumSourcefiles(codeMetrics.getMd5sumSourcefiles());
        }
        testRun.setTestDurationMillis(testDurationsMillis);
        return testRun;
    }

    /**
     * @param conn
     * @return
     * @throws SQLException
     */
    private Connection switchToTransaction(Connection conn) throws SQLException {
        conn.close();
        // Begin a transaction.
        // We can set this to a low isolation level because
        // - We don't read anything
        // - The inserts/updates we perform should not affect
        // rows visible to any other transaction
        conn = getConnection();
        conn.setAutoCommit(false);
        conn.setTransactionIsolation(Connection.TRANSACTION_READ_UNCOMMITTED);
        return conn;
    }

    /**
     * @param submissionPK
     * @param testSetupPK
     * @param testMachine
     * @param testOutcomeCollection
     */
    private void logHotspotErrors(@Submission.PK int submissionPK, int testSetupPK, String testMachine,
            TestOutcomeCollection testOutcomeCollection) {
        String adminEmail = webProperties.getProperty("admin.email");
        if (adminEmail == null || adminEmail.equals("")) {
            getSubmitServerServletLog().error("Can't find admin.email in web.properties");
        }
        String adminSMTP = webProperties.getProperty("admin.smtp");
        if (adminSMTP == null || adminSMTP.equals("")) {
            getSubmitServerServletLog().error("Can't find admin.smtp in web.properties");
        }

        for (TestOutcome outcome : testOutcomeCollection) {
            if (outcome.getLongTestResult()
                    .contains("An unexpected error has been detected by HotSpot Virtual Machine")) {
                String errorMsg = "SubmissionPK " + submissionPK + " for test-setup " + testSetupPK
                        + " from buildServer " + testMachine + " had a HotSpot exception: " + outcome;
                getSubmitServerServletLog().error(errorMsg);
                try {
                    JavaMail.sendMessage(adminEmail, adminEmail, adminSMTP, "HotSpot error!", outcome.toString());
                } catch (MessagingException e) {
                    getSubmitServerServletLog().error("Unable to email " + adminEmail + " from smtp server "
                            + adminSMTP + " a warning about a HotSpot exception on one of the buildServers", e);
                }
            }
        }
    }

    /**
     * Compares two testOutcomeCollections that were run against the same
     * testSetupPK and produces a String suitable for entry in a log4j log
     * summarizing the differences between the two collections. Used by the
     * background retesting mechanism. TODO It's ugly that this returns a String!
     * 
     * @param oldCollection
     *          The collection currently in the database.
     * @param newCollection
     *          The collection returned after a background re-test.
     * @param submissionPK
     *          The submissionPK of the submission being retested.
     * @param testSetupPK
     *          The testSetupPK of the testSetup used for the re-test.
     * @return null if the two testOutcomeCollections are the same; a String
     *         suitable for writing in a log if there are differences
     */
    private static String compareCardinalOutcomes(TestOutcomeCollection oldCollection,
            TestOutcomeCollection newCollection, @Submission.PK Integer submissionPK, Integer testSetupPK,
            String testMachine) {
        for (TestOutcome newOutcome : newCollection) {
            // If any of the tests were marked "COULD_NOT_RUN" then don't record
            // anything
            // into the DB
            if (newOutcome.getOutcome().equals(TestOutcome.COULD_NOT_RUN)) {
                getFailedBackgroundRetestLog().warn(newOutcome.getOutcome() + " result for submissionPK "
                        + submissionPK + ", testSetupPK " + testSetupPK + " and test " + newOutcome.getTestType()
                        + newOutcome.getTestNumber() + " : " + newOutcome.getTestName());
                return null;
            }
        }
        // TODO Handle when one compiles and the next doesn't compile.
        // NOTE: We're ignoring a lot of other info here like md5sums, clover
        // info, etc.
        StringBuffer buf = new StringBuffer();
        for (TestOutcome oldOutcome : oldCollection.getIterableForCardinalTestTypes()) {
            TestOutcome newOutcome = newCollection.getTest(oldOutcome.getTestName());

            if (oldOutcome.getOutcome().equals(TestOutcome.COULD_NOT_RUN)) {
                getFailedBackgroundRetestLog().warn("Not currently able to compare could not runs");
                return "skip";
            }
            if (newOutcome == null) {
                // We're in trouble here
                throw new IllegalStateException("Can't find " + oldOutcome.getTestName()
                        + " in new testOutcomeCollection after background retest for " + " submissionPK "
                        + submissionPK + " and testSetupPK " + testSetupPK);
            }
            if (!oldOutcome.getOutcome().equals(newOutcome.getOutcome())) {
                buf.append("\t" + oldOutcome.getTestName() + ": outcome in database is " + oldOutcome.getOutcome()
                        + "; background retest produced " + newOutcome.getOutcome() + "\n");
            }
        }
        if (buf.length() > 0) {
            buf.insert(0, "submissionPK = " + submissionPK + ", testSetupPK = " + testSetupPK + " on " + testMachine
                    + ":\n");
            // Strip off last newline
            buf.delete(buf.length() - 1, buf.length());
            return buf.toString();
        }
        return null;
    }
}