org.optaplanner.benchmark.impl.DefaultPlannerBenchmark.java Source code

Java tutorial

Introduction

Here is the source code for org.optaplanner.benchmark.impl.DefaultPlannerBenchmark.java

Source

/*
 * Copyright 2010 JBoss Inc
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.optaplanner.benchmark.impl;

import java.io.File;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;

import org.apache.commons.collections.comparators.ReverseComparator;
import org.drools.core.util.StringUtils;
import org.optaplanner.benchmark.api.PlannerBenchmark;
import org.optaplanner.benchmark.api.PlannerBenchmarkException;
import org.optaplanner.benchmark.api.ranking.SolverBenchmarkRankingWeightFactory;
import org.optaplanner.benchmark.impl.history.BenchmarkHistoryReport;
import org.optaplanner.benchmark.impl.report.BenchmarkReport;
import org.optaplanner.core.api.score.Score;
import org.optaplanner.core.api.solver.Solver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Represents the benchmarks on multiple {@link Solver} configurations on multiple problem instances (data sets).
 */
public class DefaultPlannerBenchmark implements PlannerBenchmark {

    protected final transient Logger logger = LoggerFactory.getLogger(getClass());

    private String name = null;
    private File benchmarkDirectory = null;
    private File benchmarkReportDirectory = null;
    private Comparator<SolverBenchmark> solverBenchmarkRankingComparator = null;
    private SolverBenchmarkRankingWeightFactory solverBenchmarkRankingWeightFactory = null;

    private int parallelBenchmarkCount = -1;
    private long warmUpTimeMillisSpend = 0L;

    private List<SolverBenchmark> solverBenchmarkList = null;
    private List<ProblemBenchmark> unifiedProblemBenchmarkList = null;
    private final BenchmarkReport benchmarkReport = new BenchmarkReport(this);

    private boolean benchmarkHistoryReportEnabled;
    private final BenchmarkHistoryReport benchmarkHistoryReport = new BenchmarkHistoryReport(this);

    private long startingSystemTimeMillis;
    private Date startingTimestamp;
    private ExecutorService executorService;
    private Integer failureCount;
    private SingleBenchmark firstFailureSingleBenchmark;

    private Long averageProblemScale = null;
    private Score averageScore = null;
    private SolverBenchmark favoriteSolverBenchmark;
    private long benchmarkTimeMillisSpend;

    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }

    public File getBenchmarkDirectory() {
        return benchmarkDirectory;
    }

    public void setBenchmarkDirectory(File benchmarkDirectory) {
        this.benchmarkDirectory = benchmarkDirectory;
    }

    public File getBenchmarkReportDirectory() {
        return benchmarkReportDirectory;
    }

    public Comparator<SolverBenchmark> getSolverBenchmarkRankingComparator() {
        return solverBenchmarkRankingComparator;
    }

    public void setSolverBenchmarkRankingComparator(Comparator<SolverBenchmark> solverBenchmarkRankingComparator) {
        this.solverBenchmarkRankingComparator = solverBenchmarkRankingComparator;
    }

    public SolverBenchmarkRankingWeightFactory getSolverBenchmarkRankingWeightFactory() {
        return solverBenchmarkRankingWeightFactory;
    }

    public void setSolverBenchmarkRankingWeightFactory(
            SolverBenchmarkRankingWeightFactory solverBenchmarkRankingWeightFactory) {
        this.solverBenchmarkRankingWeightFactory = solverBenchmarkRankingWeightFactory;
    }

    public int getParallelBenchmarkCount() {
        return parallelBenchmarkCount;
    }

    public void setParallelBenchmarkCount(int parallelBenchmarkCount) {
        this.parallelBenchmarkCount = parallelBenchmarkCount;
    }

    public long getWarmUpTimeMillisSpend() {
        return warmUpTimeMillisSpend;
    }

    public void setWarmUpTimeMillisSpend(long warmUpTimeMillisSpend) {
        this.warmUpTimeMillisSpend = warmUpTimeMillisSpend;
    }

    public List<SolverBenchmark> getSolverBenchmarkList() {
        return solverBenchmarkList;
    }

    public void setSolverBenchmarkList(List<SolverBenchmark> solverBenchmarkList) {
        this.solverBenchmarkList = solverBenchmarkList;
    }

    public List<ProblemBenchmark> getUnifiedProblemBenchmarkList() {
        return unifiedProblemBenchmarkList;
    }

    public void setUnifiedProblemBenchmarkList(List<ProblemBenchmark> unifiedProblemBenchmarkList) {
        this.unifiedProblemBenchmarkList = unifiedProblemBenchmarkList;
    }

    public BenchmarkReport getBenchmarkReport() {
        return benchmarkReport;
    }

    public boolean isBenchmarkHistoryReportEnabled() {
        return benchmarkHistoryReportEnabled;
    }

    public void setBenchmarkHistoryReportEnabled(boolean benchmarkHistoryReportEnabled) {
        this.benchmarkHistoryReportEnabled = benchmarkHistoryReportEnabled;
    }

    public BenchmarkHistoryReport getBenchmarkHistoryReport() {
        return benchmarkHistoryReport;
    }

    public Date getStartingTimestamp() {
        return startingTimestamp;
    }

    public Integer getFailureCount() {
        return failureCount;
    }

    public Long getAverageProblemScale() {
        return averageProblemScale;
    }

    public Score getAverageScore() {
        return averageScore;
    }

    public long getBenchmarkTimeMillisSpend() {
        return benchmarkTimeMillisSpend;
    }

    // ************************************************************************
    // Benchmark methods
    // ************************************************************************

    public boolean hasMultipleParallelBenchmarks() {
        return parallelBenchmarkCount > 1;
    }

    public void benchmark() {
        benchmarkingStarted();
        warmUp();
        runSingleBenchmarks();
        benchmarkingEnded();
    }

    public void benchmarkingStarted() {
        startingSystemTimeMillis = System.currentTimeMillis();
        startingTimestamp = new Date();
        if (solverBenchmarkList == null || solverBenchmarkList.isEmpty()) {
            throw new IllegalArgumentException(
                    "The solverBenchmarkList (" + solverBenchmarkList + ") cannot be empty.");
        }
        initBenchmarkDirectoryAndSubdirs();
        for (SolverBenchmark solverBenchmark : solverBenchmarkList) {
            solverBenchmark.benchmarkingStarted();
        }
        for (ProblemBenchmark problemBenchmark : unifiedProblemBenchmarkList) {
            problemBenchmark.benchmarkingStarted();
        }
        executorService = Executors.newFixedThreadPool(parallelBenchmarkCount);
        failureCount = 0;
        firstFailureSingleBenchmark = null;
        averageProblemScale = null;
        favoriteSolverBenchmark = null;
        benchmarkTimeMillisSpend = -1L;
        logger.info("Benchmarking started: solverBenchmarkList size ({}), parallelBenchmarkCount ({}).",
                solverBenchmarkList.size(), parallelBenchmarkCount);
    }

    private void initBenchmarkDirectoryAndSubdirs() {
        if (benchmarkDirectory == null) {
            throw new IllegalArgumentException(
                    "The benchmarkDirectory (" + benchmarkDirectory + ") must not be null.");
        }
        benchmarkDirectory.mkdirs();
        String timestamp = new SimpleDateFormat("yyyy-MM-dd_HHmmss").format(startingTimestamp);
        if (StringUtils.isEmpty(name)) {
            name = timestamp;
        }
        benchmarkReportDirectory = new File(benchmarkDirectory, timestamp);
        benchmarkReportDirectory.mkdirs();
    }

    private void warmUp() {
        if (warmUpTimeMillisSpend > 0L) {
            logger.info("================================================================================");
            logger.info("Warming up started");
            logger.info("================================================================================");
            long startingTimeMillis = System.currentTimeMillis();
            long timeLeft = warmUpTimeMillisSpend;
            Iterator<ProblemBenchmark> it = unifiedProblemBenchmarkList.iterator();
            while (timeLeft > 0L) {
                if (!it.hasNext()) {
                    it = unifiedProblemBenchmarkList.iterator();
                }
                ProblemBenchmark problemBenchmark = it.next();
                timeLeft = problemBenchmark.warmUp(startingTimeMillis, warmUpTimeMillisSpend, timeLeft);
            }
            logger.info("================================================================================");
            logger.info("Warming up ended");
            logger.info("================================================================================");
        }
    }

    protected void runSingleBenchmarks() {
        Map<SingleBenchmark, Future<SingleBenchmark>> futureMap = new HashMap<SingleBenchmark, Future<SingleBenchmark>>();
        for (ProblemBenchmark problemBenchmark : unifiedProblemBenchmarkList) {
            for (SingleBenchmark singleBenchmark : problemBenchmark.getSingleBenchmarkList()) {
                Future<SingleBenchmark> future = executorService.submit(singleBenchmark);
                futureMap.put(singleBenchmark, future);
            }
        }
        // wait for the benchmarks to complete
        for (Map.Entry<SingleBenchmark, Future<SingleBenchmark>> futureEntry : futureMap.entrySet()) {
            SingleBenchmark singleBenchmark = futureEntry.getKey();
            Future<SingleBenchmark> future = futureEntry.getValue();
            Throwable failureThrowable = null;
            try {
                // Explicitly returning it in the Callable guarantees memory visibility
                singleBenchmark = future.get();
                // TODO WORKAROUND Remove when PLANNER-46 is fixed.
                if (singleBenchmark.getScore() == null) {
                    throw new IllegalStateException("Score is null. TODO fix PLANNER-46.");
                }
            } catch (InterruptedException e) {
                logger.error("The singleBenchmark (" + singleBenchmark.getName() + ") was interrupted.", e);
                failureThrowable = e;
            } catch (ExecutionException e) {
                Throwable cause = e.getCause();
                logger.error("The singleBenchmark (" + singleBenchmark.getName() + ") failed.", cause);
                failureThrowable = cause;
            } catch (IllegalStateException e) {
                // TODO WORKAROUND Remove when PLANNER-46 is fixed.
                logger.error("The singleBenchmark (" + singleBenchmark.getName() + ") failed.", e);
                failureThrowable = e;
            }
            if (failureThrowable == null) {
                singleBenchmark.setSucceeded(true);
            } else {
                singleBenchmark.setSucceeded(false);
                singleBenchmark.setFailureThrowable(failureThrowable);
                failureCount++;
                if (firstFailureSingleBenchmark == null) {
                    firstFailureSingleBenchmark = singleBenchmark;
                }
            }
        }
    }

    public long calculateTimeMillisSpend() {
        long now = System.currentTimeMillis();
        return now - startingSystemTimeMillis;
    }

    public void benchmarkingEnded() {
        executorService.shutdownNow();
        for (ProblemBenchmark problemBenchmark : unifiedProblemBenchmarkList) {
            problemBenchmark.benchmarkingEnded();
        }
        for (SolverBenchmark solverBenchmark : solverBenchmarkList) {
            solverBenchmark.benchmarkingEnded();
        }
        determineTotalsAndAverages();
        determineSolverBenchmarkRanking();
        benchmarkTimeMillisSpend = calculateTimeMillisSpend();
        benchmarkReport.writeReport();
        if (benchmarkHistoryReportEnabled) {
            benchmarkHistoryReport.writeHistory();
        }
        if (failureCount == 0) {
            logger.info(
                    "Benchmarking ended: time spend ({}), favoriteSolverBenchmark ({}), statistic html overview ({}).",
                    benchmarkTimeMillisSpend, favoriteSolverBenchmark.getName(),
                    benchmarkReport.getHtmlOverviewFile().getAbsolutePath());
        } else {
            logger.info("Benchmarking failed: time spend ({}), failureCount ({}), statistic html overview ({}).",
                    benchmarkTimeMillisSpend, failureCount,
                    benchmarkReport.getHtmlOverviewFile().getAbsolutePath());
            throw new PlannerBenchmarkException(
                    "Benchmarking failed: failureCount (" + failureCount + ")."
                            + " The exception of the firstFailureSingleBenchmark ("
                            + firstFailureSingleBenchmark.getName() + ") is chained.",
                    firstFailureSingleBenchmark.getFailureThrowable());
        }
    }

    private void determineTotalsAndAverages() {
        long totalProblemScale = 0L;
        int problemScaleCount = 0;
        for (ProblemBenchmark problemBenchmark : unifiedProblemBenchmarkList) {
            Long problemScale = problemBenchmark.getProblemScale();
            if (problemScale != null && problemScale >= 0L) {
                totalProblemScale += problemScale;
                problemScaleCount++;
            }
        }
        averageProblemScale = problemScaleCount == 0 ? null : totalProblemScale / (long) problemScaleCount;
        Score totalScore = null;
        int solverBenchmarkCount = 0;
        for (SolverBenchmark solverBenchmark : solverBenchmarkList) {
            Score score = solverBenchmark.getAverageScore();
            if (score != null) {
                totalScore = (totalScore == null) ? score : totalScore.add(score);
                solverBenchmarkCount++;
            }
        }
        if (totalScore != null) {
            averageScore = totalScore.divide(solverBenchmarkCount);
        }
    }

    private void determineSolverBenchmarkRanking() {
        List<SolverBenchmark> rankableSolverBenchmarkList = new ArrayList<SolverBenchmark>(solverBenchmarkList);
        // Do not rank a SolverBenchmark that has a failure
        for (Iterator<SolverBenchmark> it = rankableSolverBenchmarkList.iterator(); it.hasNext();) {
            SolverBenchmark solverBenchmark = it.next();
            if (solverBenchmark.hasAnyFailure()) {
                it.remove();
            }
        }
        List<List<SolverBenchmark>> sameRankingListList = createSameRankingListList(rankableSolverBenchmarkList);
        int ranking = 0;
        for (List<SolverBenchmark> sameRankingList : sameRankingListList) {
            for (SolverBenchmark solverBenchmark : sameRankingList) {
                solverBenchmark.setRanking(ranking);
            }
            ranking += sameRankingList.size();
        }
        favoriteSolverBenchmark = sameRankingListList.isEmpty() ? null : sameRankingListList.get(0).get(0);
    }

    private List<List<SolverBenchmark>> createSameRankingListList(
            List<SolverBenchmark> rankableSolverBenchmarkList) {
        List<List<SolverBenchmark>> sameRankingListList = new ArrayList<List<SolverBenchmark>>(
                rankableSolverBenchmarkList.size());
        if (solverBenchmarkRankingComparator != null) {
            Comparator<SolverBenchmark> comparator = Collections.reverseOrder(solverBenchmarkRankingComparator);
            Collections.sort(rankableSolverBenchmarkList, comparator);
            List<SolverBenchmark> sameRankingList = null;
            SolverBenchmark previousSolverBenchmark = null;
            for (SolverBenchmark solverBenchmark : rankableSolverBenchmarkList) {
                if (previousSolverBenchmark == null
                        || comparator.compare(previousSolverBenchmark, solverBenchmark) != 0) {
                    // New rank
                    sameRankingList = new ArrayList<SolverBenchmark>();
                    sameRankingListList.add(sameRankingList);
                }
                sameRankingList.add(solverBenchmark);
                previousSolverBenchmark = solverBenchmark;
            }
        } else if (solverBenchmarkRankingWeightFactory != null) {
            SortedMap<Comparable, List<SolverBenchmark>> rankedMap = new TreeMap<Comparable, List<SolverBenchmark>>(
                    new ReverseComparator());
            for (SolverBenchmark solverBenchmark : rankableSolverBenchmarkList) {
                Comparable rankingWeight = solverBenchmarkRankingWeightFactory
                        .createRankingWeight(rankableSolverBenchmarkList, solverBenchmark);
                List<SolverBenchmark> sameRankingList = rankedMap.get(rankingWeight);
                if (sameRankingList == null) {
                    sameRankingList = new ArrayList<SolverBenchmark>();
                    rankedMap.put(rankingWeight, sameRankingList);
                }
                sameRankingList.add(solverBenchmark);
            }
            for (Map.Entry<Comparable, List<SolverBenchmark>> entry : rankedMap.entrySet()) {
                sameRankingListList.add(entry.getValue());
            }
        } else {
            throw new IllegalStateException("Ranking is impossible"
                    + " because solverBenchmarkRankingComparator and solverBenchmarkRankingWeightFactory are null.");
        }
        return sameRankingListList;
    }

    public boolean hasAnyFailure() {
        return failureCount > 0;
    }

    // TODO Temporarily disabled because it crashes because of http://jira.codehaus.org/browse/XSTR-666
    //    public void writeBenchmarkResult(XStream xStream) {
    //        File benchmarkResultFile = new File(benchmarkReportDirectory, "benchmarkResult.xml");
    //        OutputStreamWriter writer = null;
    //        try {
    //            writer = new OutputStreamWriter(new FileOutputStream(benchmarkResultFile), "UTF-8");
    //            xStream.toXML(this, writer);
    //        } catch (UnsupportedEncodingException e) {
    //            throw new IllegalStateException("This JVM does not support UTF-8 encoding.", e);
    //        } catch (FileNotFoundException e) {
    //            throw new IllegalArgumentException(
    //                    "Could not create benchmarkResultFile (" + benchmarkResultFile + ").", e);
    //        } finally {
    //            IOUtils.closeQuietly(writer);
    //        }
    //    }

    @Override
    public String toString() {
        return getName();
    }

}