Java tutorial
/** * Copyright 2008 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.sf.katta.tool.loadtest; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.Writer; import java.util.ArrayList; import java.util.List; import net.sf.katta.master.MasterContext; import net.sf.katta.operation.OperationId; import net.sf.katta.operation.master.MasterOperation; import net.sf.katta.operation.node.OperationResult; import net.sf.katta.protocol.InteractionProtocol; import net.sf.katta.tool.loadtest.query.AbstractQueryExecutor; import org.apache.commons.math.stat.descriptive.StorelessUnivariateStatistic; import org.apache.commons.math.stat.descriptive.moment.Mean; import org.apache.commons.math.stat.descriptive.moment.StandardDeviation; import org.apache.log4j.Logger; @SuppressWarnings("serial") public class LoadTestMasterOperation implements MasterOperation { private static final Logger LOG = Logger.getLogger(LoadTestMasterOperation.class); private String _masterName; private final int _numberOfTesterNodes; private final int _startRate; private final int _endRate; private final int _step; private final long _runTime; private final AbstractQueryExecutor _queryExecutor; private final File _resultDir; private final long _startTime; private int _currentIteration; private long _currentIterationStartTime; public LoadTestMasterOperation(int nodes, int startRate, int endRate, int step, long runTime, AbstractQueryExecutor queryExecutor, File resultDir) { _numberOfTesterNodes = nodes; _startRate = startRate; _endRate = endRate; _step = step; _queryExecutor = queryExecutor; _runTime = runTime; _resultDir = resultDir; _startTime = System.currentTimeMillis(); } @Override public ExecutionInstruction getExecutionInstruction(List<MasterOperation> runningOperations) throws Exception { return ExecutionInstruction.EXECUTE; } private String getName() { return "load-test-" + _startTime; } public void registerCompletion(InteractionProtocol protocol) { protocol.setFlag(getName()); } public void joinCompletion(InteractionProtocol protocol) throws InterruptedException { while (protocol.flagExists(getName())) { Thread.sleep(1000); } } @Override public List<OperationId> execute(MasterContext context, List<MasterOperation> runningOperations) throws Exception { _currentIterationStartTime = System.currentTimeMillis(); if (_masterName == null) { _masterName = context.getMaster().getMasterName(); _resultDir.mkdirs(); if (!_resultDir.isDirectory()) { throw new IllegalStateException( "result dir '" + _resultDir.getAbsolutePath() + "' cannot be created"); } } else if (!_masterName.equals(context.getMaster().getMasterName())) { throw new IllegalStateException( "master change detected - load test not safe for this since it writes local log files!"); } List<String> testNodes = context.getProtocol().getLiveNodes(); if (testNodes.size() < _numberOfTesterNodes) { throw new IllegalStateException( "only " + testNodes.size() + " available, needing " + _numberOfTesterNodes); } testNodes = testNodes.subList(0, _numberOfTesterNodes); final int queryRate = calculateCurrentQueryRate(); if (_currentIteration == 0) { LOG.info("starting load test with " + testNodes.size() + " nodes"); } LOG.info("executing tests in iteration " + _currentIteration + " at query rate: " + queryRate + " queries per second and with a run time of " + _runTime / 1000 + " seconds"); int remainingQueryRate = queryRate; int remainingNodes = testNodes.size(); List<OperationId> nodeOperationIds = new ArrayList<OperationId>(); for (String testNode : testNodes) { int queryRateForNode = remainingQueryRate / remainingNodes; LOG.info("instructing test on node " + testNode + " using query rate: " + queryRateForNode + " queries per second."); LoadTestNodeOperation nodeOperation = new LoadTestNodeOperation(_queryExecutor, queryRateForNode, _runTime); --remainingNodes; remainingQueryRate -= queryRateForNode; OperationId operationId = context.getProtocol().addNodeOperation(testNode, nodeOperation); nodeOperationIds.add(operationId); } return nodeOperationIds; } private int calculateCurrentQueryRate() { return _startRate + _currentIteration * _step; } @Override public void nodeOperationsComplete(MasterContext context, List<OperationResult> nodeResults) throws Exception { try { final int queryRate = calculateCurrentQueryRate(); LOG.info("collecting results for iteration " + _currentIteration + " and query rate " + queryRate + " after " + (System.currentTimeMillis() - _currentIterationStartTime) + " ms ..."); List<LoadTestQueryResult> queryResults = new ArrayList<LoadTestQueryResult>(); for (OperationResult operationResult : nodeResults) { if (operationResult == null || operationResult.getUnhandledException() != null) { Exception rootException = null; if (operationResult != null) { rootException = operationResult.getUnhandledException(); } throw new IllegalStateException( "at least one node operation did not completed properly: " + nodeResults, rootException); } LoadTestNodeOperationResult nodeOperationResult = (LoadTestNodeOperationResult) operationResult; queryResults.addAll(nodeOperationResult.getQueryResults()); } LOG.info("Received " + queryResults.size() + " queries, expected " + queryRate * _runTime / 1000); File statisticsFile = new File(_resultDir, "load-test-log-" + _startTime + ".log"); File resultsFile = new File(_resultDir, "load-test-results-" + _startTime + ".log"); Writer statisticsWriter = new OutputStreamWriter(new FileOutputStream(statisticsFile, true)); Writer resultWriter = new OutputStreamWriter(new FileOutputStream(resultsFile, true)); if (_currentIteration == 0) { // print headers statisticsWriter.append("#queryRate \tnode \tstartTime \tendTime \telapseTime \tquery \n"); resultWriter.append( "#requestedQueryRate \tachievedQueryRate \tfiredQueries \tqueryErrors \tavarageQueryDuration \tstandardDeviation \n"); } try { StorelessUnivariateStatistic timeStandardDeviation = new StandardDeviation(); StorelessUnivariateStatistic timeMean = new Mean(); int errors = 0; for (LoadTestQueryResult result : queryResults) { long elapsedTime = result.getEndTime() > 0 ? result.getEndTime() - result.getStartTime() : -1; statisticsWriter.write(queryRate + "\t" + result.getNodeId() + "\t" + result.getStartTime() + "\t" + result.getEndTime() + "\t" + elapsedTime + "\t" + result.getQuery() + "\n"); if (elapsedTime != -1) { timeStandardDeviation.increment(elapsedTime); timeMean.increment(elapsedTime); } else { ++errors; } } resultWriter.write(queryRate + "\t" + ((double) queryResults.size() / (_runTime / 1000)) + "\t" + queryResults.size() + "\t" + errors + "\t" + (int) timeMean.getResult() + "\t" + (int) timeStandardDeviation.getResult() + "\n"); } catch (IOException e) { throw new IllegalStateException("Failed to write statistics data.", e); } try { LOG.info("results written to " + resultsFile.getAbsolutePath()); LOG.info("statistics written to " + statisticsFile.getAbsolutePath()); statisticsWriter.close(); resultWriter.close(); } catch (IOException e) { LOG.warn("Failed to close statistics file."); } if (queryRate + _step <= _endRate) { _currentIteration++; LOG.info("triggering next iteration " + _currentIteration); context.getMasterQueue().add(this); } else { LOG.info("finish load test in iteration " + _currentIteration + " after " + (System.currentTimeMillis() - _startTime) + " ms"); context.getProtocol().removeFlag(getName()); } } catch (Exception e) { context.getProtocol().removeFlag(getName()); } } }