hu.bme.mit.sette.common.tasks.TestSuiteRunner.java Source code

Java tutorial

Introduction

Here is the source code for hu.bme.mit.sette.common.tasks.TestSuiteRunner.java

Source

/*
 * SETTE - Symbolic Execution based Test Tool Evaluator
 *
 * SETTE is a tool to help the evaluation and comparison of symbolic execution
 * based test input generator tools.
 *
 * Budapest University of Technology and Economics (BME)
 *
 * Authors: Lajos Cseppent <lajos.cseppento@inf.mit.bme.hu>, Zoltn Micskei
 * <micskeiz@mit.bme.hu>
 *
 * Copyright 2014
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */
package hu.bme.mit.sette.common.tasks;

import hu.bme.mit.sette.common.Tool;
import hu.bme.mit.sette.common.exceptions.SetteGeneralException;
import hu.bme.mit.sette.common.model.runner.ResultType;
import hu.bme.mit.sette.common.model.runner.RunnerProjectUtils;
import hu.bme.mit.sette.common.model.runner.xml.FileCoverageElement;
import hu.bme.mit.sette.common.model.runner.xml.SnippetCoverageXml;
import hu.bme.mit.sette.common.model.runner.xml.SnippetElement;
import hu.bme.mit.sette.common.model.runner.xml.SnippetInputsXml;
import hu.bme.mit.sette.common.model.runner.xml.SnippetProjectElement;
import hu.bme.mit.sette.common.model.snippet.Snippet;
import hu.bme.mit.sette.common.model.snippet.SnippetContainer;
import hu.bme.mit.sette.common.model.snippet.SnippetProject;

import java.io.File;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;

import junit.framework.AssertionFailedError;
import junit.framework.TestCase;

import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Triple;
import org.jacoco.core.analysis.Analyzer;
import org.jacoco.core.analysis.CoverageBuilder;
import org.jacoco.core.analysis.IClassCoverage;
import org.jacoco.core.analysis.ICounter;
import org.jacoco.core.data.ExecutionDataStore;
import org.jacoco.core.data.SessionInfoStore;
import org.jacoco.core.instr.Instrumenter;
import org.jacoco.core.runtime.IRuntime;
import org.jacoco.core.runtime.LoggerRuntime;
import org.jacoco.core.runtime.RuntimeData;
import org.simpleframework.xml.Serializer;
import org.simpleframework.xml.convert.AnnotationStrategy;
import org.simpleframework.xml.core.Persister;
import org.simpleframework.xml.stream.Format;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class TestSuiteRunner extends SetteTask<Tool> {
    private static final Logger logger = LoggerFactory.getLogger(TestSuiteRunner.class);

    public TestSuiteRunner(final SnippetProject snippetProject, final File outputDirectory, final Tool tool) {
        super(snippetProject, outputDirectory, tool);
    }

    public void analyze() throws Exception {
        if (!RunnerProjectUtils.getRunnerLogFile(getRunnerProjectSettings()).exists()) {
            throw new SetteGeneralException(
                    "Run the tool on the runner project first (and then parse and generate tests)");
        }

        File testsDir = getRunnerProjectSettings().getTestsDirectory();
        if (!testsDir.exists()) {
            throw new SetteGeneralException("tests dir does not exist");
        }

        Serializer serializer = new Persister(new AnnotationStrategy());

        // binary directories for the JaCoCoClassLoader
        File[] binaryDirectories = new File[2];
        binaryDirectories[0] = getSnippetProject().getSettings().getSnippetBinaryDirectory();
        binaryDirectories[1] = getRunnerProjectSettings().getBinaryDirectory();
        logger.debug("Binary directories: {}", (Object) binaryDirectories);

        // foreach containers
        for (SnippetContainer container : getSnippetProject().getModel().getContainers()) {
            // skip container with higher java version than supported
            if (container.getRequiredJavaVersion().compareTo(getTool().getSupportedJavaVersion()) > 0) {
                // TODO error handling
                System.err.println("Skipping container: " + container.getJavaClass().getName()
                        + " (required Java version: " + container.getRequiredJavaVersion() + ")");
                continue;
            }

            // foreach snippets
            for (Snippet snippet : container.getSnippets().values()) {
                File inputsXmlFile = RunnerProjectUtils.getSnippetInputsFile(getRunnerProjectSettings(), snippet);

                if (!inputsXmlFile.exists()) {
                    System.err.println("Missing: " + inputsXmlFile);
                    continue;
                }

                // save current class loader
                ClassLoader originalClassLoader = Thread.currentThread().getContextClassLoader();

                // set snippet project class loader
                Thread.currentThread().setContextClassLoader(getSnippetProject().getClassLoader());

                // read data
                SnippetInputsXml inputsXml = serializer.read(SnippetInputsXml.class, inputsXmlFile);

                // set back the original class loader
                Thread.currentThread().setContextClassLoader(originalClassLoader);

                if (inputsXml.getResultType() != ResultType.S && inputsXml.getResultType() != ResultType.C
                        && inputsXml.getResultType() != ResultType.NC) {
                    // skip!
                    continue;
                }

                if (inputsXml.getGeneratedInputs().size() == 0) {
                    System.err.println("No inputs: " + inputsXmlFile);
                }

                // toto remove try-catch
                try {
                    analyzeOne(snippet, binaryDirectories);
                } catch (Exception e) {
                    // now dump and go on
                    e.printStackTrace();
                }
            }
        }

        // TODO remove debug
        System.err.println("=> ANALYZE ENDED");
    }

    private void analyzeOne(Snippet snippet, File[] binaryDirectories) throws Exception {
        String snippetClassName = snippet.getContainer().getJavaClass().getName();
        String snippetMethodName = snippet.getMethod().getName();
        String testClassName = snippet.getContainer().getJavaClass().getName() + "_" + snippet.getMethod().getName()
                + "_Tests";

        logger.debug("Snippet: {}#{}()", snippetClassName, snippetMethodName);
        logger.debug("Tests: {}", testClassName);

        // create JaCoCo runtime and instrumenter
        IRuntime runtime = new LoggerRuntime();
        Instrumenter instrumenter = new Instrumenter(runtime);

        // start runtime
        RuntimeData data = new RuntimeData();
        runtime.startup(data);

        // create class loader

        JaCoCoClassLoader classLoader = new JaCoCoClassLoader(binaryDirectories, instrumenter,
                getSnippetProject().getClassLoader());

        // load test class
        // snippet class and other dependencies will be loaded and instrumented
        // on the fly
        Class<?> testClass = classLoader.loadClass(testClassName);

        TestCase testClassInstance = (TestCase) testClass.newInstance();

        // invoke test methods
        // TODO separate collect and invoke
        for (Method m : testClass.getDeclaredMethods()) {
            if (m.isSynthetic()) {
                // skip synthetic method
                continue;
            }

            if (m.getName().startsWith("test")) {
                logger.trace("Invoking: " + m.getName());
                try {
                    m.invoke(testClassInstance);
                } catch (InvocationTargetException e) {
                    Throwable cause = e.getCause();

                    if (cause instanceof NullPointerException || cause instanceof ArrayIndexOutOfBoundsException
                            || cause instanceof AssertionFailedError) {
                        logger.warn(cause.getClass().getName() + ": " + m.getDeclaringClass().getName() + "."
                                + m.getName());
                    } else {
                        logger.error("Exception: " + m.getDeclaringClass().getName() + "." + m.getName());
                    }
                    logger.debug(e.getMessage(), e);
                }
            } else {
                logger.warn("Not test method: {}", m.getName());
            }
        }

        // collect data
        ExecutionDataStore executionData = new ExecutionDataStore();
        SessionInfoStore sessionInfos = new SessionInfoStore();
        data.collect(executionData, sessionInfos, false);
        runtime.shutdown();

        // get classes to analyse
        // store string to avoid the mess up between the different class loaders
        Set<String> javaClasses = new HashSet<>();
        javaClasses.add(snippetClassName);

        for (Constructor<?> inclConstructor : snippet.getIncludedConstructors()) {
            javaClasses.add(inclConstructor.getDeclaringClass().getName());
        }

        for (Method inclMethod : snippet.getIncludedMethods()) {
            javaClasses.add(inclMethod.getDeclaringClass().getName());
        }

        // TODO inner classes are not handled well enough

        // TODO anonymous classes can also have anonymous classes -> recursion

        Set<String> toAdd = new HashSet<>();
        for (String javaClass : javaClasses) {
            int i = 1;
            while (true) {
                // guess anonymous classes, like ClassName$1, ClassName$2 etc.
                try {
                    classLoader.loadClass(javaClass + "$" + i);
                    toAdd.add(javaClass + "$" + i);
                    i++;
                } catch (ClassNotFoundException e) {
                    // bad guess, no more anonymous classes on this level
                    break;
                }
            }
        }
        javaClasses.addAll(toAdd);

        // analyse classes
        CoverageBuilder coverageBuilder = new CoverageBuilder();
        Analyzer analyzer = new Analyzer(executionData, coverageBuilder);

        for (String javaClassName : javaClasses) {
            logger.trace("Analysing: {}", javaClassName);
            analyzer.analyzeClass(classLoader.readBytes(javaClassName), javaClassName);
        }

        // TODO remove debug
        // new File("D:/SETTE/!DUMP/" + getTool().getName()).mkdirs();
        // PrintStream out = new PrintStream("D:/SETTE/!DUMP/"
        // + getTool().getName() + "/" + testClassName + ".out");

        Map<String, Triple<TreeSet<Integer>, TreeSet<Integer>, TreeSet<Integer>>> coverageInfo = new HashMap<>();

        for (final IClassCoverage cc : coverageBuilder.getClasses()) {
            String file = cc.getPackageName() + '/' + cc.getSourceFileName();
            file = file.replace('\\', '/');

            if (!coverageInfo.containsKey(file)) {
                coverageInfo.put(file,
                        Triple.of(new TreeSet<Integer>(), new TreeSet<Integer>(), new TreeSet<Integer>()));
            }

            // out.printf("Coverage of class %s%n", cc.getName());
            //
            // printCounter(out, "instructions",
            // cc.getInstructionCounter());
            // printCounter(out, "branches", cc.getBranchCounter());
            // printCounter(out, "lines", cc.getLineCounter());
            // printCounter(out, "methods", cc.getMethodCounter());
            // printCounter(out, "complexity", cc.getComplexityCounter());

            for (int l = cc.getFirstLine(); l <= cc.getLastLine(); l++) {
                switch (cc.getLine(l).getStatus()) {
                case ICounter.FULLY_COVERED:
                    coverageInfo.get(file).getLeft().add(l);
                    break;

                case ICounter.PARTLY_COVERED:
                    coverageInfo.get(file).getMiddle().add(l);
                    break;

                case ICounter.NOT_COVERED:
                    coverageInfo.get(file).getRight().add(l);
                    break;
                }
            }
        }

        // create coverage XML
        SnippetCoverageXml coverageXml = new SnippetCoverageXml();
        coverageXml.setToolName(getTool().getName());
        coverageXml.setSnippetProjectElement(
                new SnippetProjectElement(getSnippetProjectSettings().getBaseDirectory().getCanonicalPath()));

        coverageXml.setSnippetElement(
                new SnippetElement(snippet.getContainer().getJavaClass().getName(), snippet.getMethod().getName()));

        coverageXml.setResultType(ResultType.S);

        for (Entry<String, Triple<TreeSet<Integer>, TreeSet<Integer>, TreeSet<Integer>>> entry : coverageInfo
                .entrySet()) {
            TreeSet<Integer> full = entry.getValue().getLeft();
            TreeSet<Integer> partial = entry.getValue().getMiddle();
            TreeSet<Integer> not = entry.getValue().getRight();

            FileCoverageElement fce = new FileCoverageElement();
            fce.setName(entry.getKey());
            fce.setFullyCoveredLines(StringUtils.join(full, ' '));
            fce.setPartiallyCoveredLines(StringUtils.join(partial, ' '));
            fce.setNotCoveredLines(StringUtils.join(not, ' '));

            coverageXml.getCoverage().add(fce);
        }

        coverageXml.validate();

        // TODO needs more documentation
        File coverageFile = RunnerProjectUtils.getSnippetCoverageFile(getRunnerProjectSettings(), snippet);

        Serializer serializer = new Persister(new AnnotationStrategy(),
                new Format("<?xml version=\"1.0\" encoding= \"UTF-8\" ?>"));

        serializer.write(coverageXml, coverageFile);

        // TODO move HTML generation to another file/phase
        File htmlFile = RunnerProjectUtils.getSnippetHtmlFile(getRunnerProjectSettings(), snippet);

        String htmlTitle = getTool().getName() + " - " + snippetClassName + '.' + snippetMethodName + "()";
        StringBuilder htmlData = new StringBuilder();
        htmlData.append("<!DOCTYPE html>\n");
        htmlData.append("<html lang=\"hu\">\n");
        htmlData.append("<head>\n");
        htmlData.append("       <meta charset=\"utf-8\" />\n");
        htmlData.append("       <title>" + htmlTitle + "</title>\n");
        htmlData.append("       <style type=\"text/css\">\n");
        htmlData.append("               .code { font-family: 'Consolas', monospace; }\n");
        htmlData.append("               .code .line { border-bottom: 1px dotted #aaa; white-space: pre; }\n");
        htmlData.append("               .code .green { background-color: #CCFFCC; }\n");
        htmlData.append("               .code .yellow { background-color: #FFFF99; }\n");
        htmlData.append("               .code .red { background-color: #FFCCCC; }\n");
        htmlData.append("               .code .line .number {\n");
        htmlData.append("                       display: inline-block;\n");
        htmlData.append("                       width:50px;\n");
        htmlData.append("                       text-align:right;\n");
        htmlData.append("                       margin-right:5px;\n");
        htmlData.append("               }\n");
        htmlData.append("       </style>\n");
        htmlData.append("</head>\n");
        htmlData.append("\n");
        htmlData.append("<body>\n");
        htmlData.append("       <h1>" + htmlTitle + "</h1>\n");

        for (FileCoverageElement fce : coverageXml.getCoverage()) {
            htmlData.append("       <h2>" + fce.getName() + "</h2>\n");
            htmlData.append("       \n");

            File src = new File(getSnippetProject().getSettings().getSnippetSourceDirectory(), fce.getName());
            List<String> srcLines = FileUtils.readLines(src);

            SortedSet<Integer> full = linesToSortedSet(fce.getFullyCoveredLines());
            SortedSet<Integer> partial = linesToSortedSet(fce.getPartiallyCoveredLines());
            SortedSet<Integer> not = linesToSortedSet(fce.getNotCoveredLines());

            htmlData.append("       <div class=\"code\">\n");
            int i = 1;
            for (String srcLine : srcLines) {
                String divClass = getLineDivClass(i, full, partial, not);
                htmlData.append("               <div class=\"" + divClass + "\"><div class=\"number\">" + i
                        + "</div> " + srcLine + "</div>\n");
                i++;
            }
            htmlData.append("       </div>\n\n");

        }

        // htmlData.append("               <div class=\"line\"><div class=\"number\">1</div> package samplesnippets;</div>\n");
        // htmlData.append("               <div class=\"line\"><div class=\"number\">2</div> </div>\n");
        // htmlData.append("               <div class=\"line\"><div class=\"number\">3</div> import hu.bme.mit.sette.annotations.SetteIncludeCoverage;</div>\n");
        // htmlData.append("               <div class=\"line\"><div class=\"number\">4</div> import hu.bme.mit.sette.annotations.SetteNotSnippet;</div>\n");
        // htmlData.append("               <div class=\"line\"><div class=\"number\">5</div> import hu.bme.mit.sette.annotations.SetteRequiredStatementCoverage;</div>\n");
        // htmlData.append("               <div class=\"line\"><div class=\"number\">6</div> import hu.bme.mit.sette.annotations.SetteSnippetContainer;</div>\n");
        // htmlData.append("               <div class=\"line\"><div class=\"number\">7</div> import samplesnippets.inputs.SampleContainer_Inputs;</div>\n");
        // htmlData.append("               <div class=\"line\"><div class=\"number\">8</div> </div>\n");
        // htmlData.append("               <div class=\"line\"><div class=\"number\">9</div> @SetteSnippetContainer(category = "X1",</div>\n");
        // htmlData.append("               <div class=\"line\"><div class=\"number\">10</div>         goal = "Sample
        // snippet container",</div>\n");
        // htmlData.append("               <div class=\"line\"><div class=\"number\">11</div>         inputFactoryContainer = SampleContainer_Inputs.class)</div>\n");
        // htmlData.append("               <div class=\"line\"><div class=\"number\">12</div> public final class SampleContainer {</div>\n");
        // htmlData.append("               <div class=\"line\"><div class=\"number\">13</div>     private SampleContainer() {</div>\n");
        // htmlData.append("               <div class=\"line red\"><div class=\"number\">14</div>         throw new UnsupportedOperationException("Static
        // class");</div>\n");
        // htmlData.append("               <div class=\"line\"><div class=\"number\">15</div>     }</div>\n");
        // htmlData.append("               <div class=\"line\"><div class=\"number\">16</div> </div>\n");
        // htmlData.append("               <div class=\"line\"><div class=\"number\">17</div>     @SetteRequiredStatementCoverage(value = 100)</div>\n");
        // htmlData.append("               <div class=\"line green\"><div class=\"number\">18</div>     public static boolean snippet(int x) {</div>\n");
        // htmlData.append("               <div class=\"line yellow\"><div class=\"number\">19</div>         if (20 * x + 2 == 42) {</div>\n");
        // htmlData.append("               <div class=\"line green\"><div class=\"number\">20</div>             return true;</div>\n");
        // htmlData.append("               <div class=\"line green\"><div class=\"number\">21</div>         } else {</div>\n");
        // htmlData.append("               <div class=\"line green\"><div class=\"number\">22</div>             return false;</div>\n");
        // htmlData.append("               <div class=\"line green\"><div class=\"number\">23</div>         }</div>\n");
        // htmlData.append("               <div class=\"line green\"><div class=\"number\">24</div>     }</div>\n");
        // htmlData.append("               <div class=\"line\"><div class=\"number\">25</div> }</div>\n");
        // htmlData.append("               <div class=\"line\"><div class=\"number\">26</div> </div>\n");

        htmlData.append("</body>\n");
        htmlData.append("</html>\n");

        FileUtils.write(htmlFile, htmlData);
    }

    // TODO needed anymore?
    // private void printCounter(PrintStream out, final String unit,
    // final ICounter counter) {
    // final Integer missed = Integer
    // .valueOf(counter.getMissedCount());
    // final Integer total = Integer.valueOf(counter.getTotalCount());
    // out.printf("%s of %s %s missed%n", missed, total, unit);
    // }

    private SortedSet<Integer> linesToSortedSet(String lines) {
        SortedSet<Integer> sortedSet = new TreeSet<>();

        for (String line : lines.split("\\s+")) {
            if (StringUtils.isBlank(line)) {
                continue;
            }

            sortedSet.add(Integer.parseInt(line));
        }

        return sortedSet;
    }

    private String getLineDivClass(int i, SortedSet<Integer> full, SortedSet<Integer> partial,
            SortedSet<Integer> not) {
        if (full.contains(i)) {
            return "line green";
        } else if (partial.contains(i)) {
            return "line yellow";
        } else if (not.contains(i)) {
            return "line red";
        } else {
            return "line";
        }
    }
}