Java tutorial
/* * Copyright 2010 Ning, Inc. * * Ning licenses this file to you under the Apache License, version 2.0 * (the "License"); you may not use this file except in compliance with the * License. You may obtain a copy of the License at: * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations * under the License. */ package com.ning.metrics.collector; import com.ning.metrics.collector.util.NamedThreadFactory; import com.ning.metrics.serialization.event.SmileEnvelopeEvent; import com.ning.metrics.serialization.thrift.ThriftField; import com.ning.metrics.serialization.thrift.ThriftFieldListSerializer; import org.apache.commons.codec.binary.Base64; import org.apache.log4j.Logger; import org.apache.thrift.TException; import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.transport.TFramedTransport; import org.apache.thrift.transport.TSocket; import org.apache.thrift.transport.TTransportException; import org.codehaus.jackson.JsonGenerator; import org.codehaus.jackson.smile.SmileFactory; import org.joda.time.DateTime; import scribe.thrift.LogEntry; import scribe.thrift.ResultCode; import scribe.thrift.scribe; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.ArrayList; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; public class TestPerformance { private final static int THREADPOOL_SIZE = 5; private final static int NUMBER_OF_SCRIBE_CLIENTS = 5; private final static int NUMBER_OF_MESSAGES_PER_SCRIBE_PAYLOAD = 60; private static int NUMBER_OF_MESSAGES_PER_SCRIBE_CLIENT; private static ArrayList<LogEntry> messages; private static scribe.Client client; private final static Logger log = Logger.getLogger(TestPerformance.class); static final private Object lockObject = new Object(); private static class ScribeClient implements Runnable { /** * When an object implementing interface <code>Runnable</code> is used * to create a thread, starting the thread causes the object's * <code>run</code> method to be called in that separately executing * thread. * <p/> * The general contract of the method <code>run</code> is that it may * take any action whatsoever. * * @see Thread#run() */ @Override public void run() { ResultCode rescode = null; for (int i = 0; i < NUMBER_OF_MESSAGES_PER_SCRIBE_CLIENT; i++) { try { synchronized (lockObject) { // Not Thread-safe. Sigh. Really Facebook? rescode = client.Log(messages); } } catch (TException e) { log.warn(e.getLocalizedMessage()); } switch (rescode) { case OK: log.debug(String.format("%d messages sent successfully.", messages.size())); break; case TRY_LATER: log.error("Falling over"); System.exit(1); break; default: log.warn(String.format("Unknown error: %d", rescode.getValue())); } } } } public static void main(String[] args) throws Exception { final TFramedTransport transport = createScribeClient(); String message = createThriftPayload(); //String message = createSmilePayload(); NUMBER_OF_MESSAGES_PER_SCRIBE_CLIENT = 2000; doOneRun(message); //NUMBER_OF_MESSAGES_PER_SCRIBE_CLIENT = 2500; //doOneRun(message); //NUMBER_OF_MESSAGES_PER_SCRIBE_CLIENT = 3000; //doOneRun(message); //NUMBER_OF_MESSAGES_PER_SCRIBE_CLIENT = 20000; //doOneRun(message); transport.close(); System.exit(0); } private static void doOneRun(String message) throws InterruptedException { messages = new ArrayList<LogEntry>(NUMBER_OF_MESSAGES_PER_SCRIBE_PAYLOAD); for (int i = 0; i < NUMBER_OF_MESSAGES_PER_SCRIBE_PAYLOAD; i++) { messages.add(i, new LogEntry("category", message)); } log.debug("Scribe payload created"); int messageSize = message.length(); long startTime = scheduleScribeAgents(); final long runTimeSeconds = (System.currentTimeMillis() - startTime) / 1000; //final double throughput = 8 / 1024. * messageSize * NUMBER_OF_SCRIBE_CLIENTS * NUMBER_OF_MESSAGES_PER_SCRIBE_PAYLOAD * NUMBER_OF_MESSAGES_PER_SCRIBE_CLIENT / runTimeSeconds; // TODO troughput meaningless in its current form (what payload?) //log.error(String.format("%d messages sent in %d:%02d, %s bytes per payload, %.4f Mb/sec throughput", log.error(String.format("%d messages sent in %d:%02d, %s bytes per payload", NUMBER_OF_SCRIBE_CLIENTS * NUMBER_OF_MESSAGES_PER_SCRIBE_PAYLOAD * NUMBER_OF_MESSAGES_PER_SCRIBE_CLIENT, runTimeSeconds / 60, runTimeSeconds % 60, messageSize)); } private static long scheduleScribeAgents() throws InterruptedException { ExecutorService e = Executors.newFixedThreadPool(THREADPOOL_SIZE, new NamedThreadFactory("Performance tests (Scribe client)")); long startTime = System.currentTimeMillis(); for (int i = 0; i < NUMBER_OF_SCRIBE_CLIENTS; i++) { e.execute(new ScribeClient()); log.debug(String.format("Thread %d/%d submitted", i + 1, NUMBER_OF_SCRIBE_CLIENTS)); } e.shutdown(); e.awaitTermination(10, TimeUnit.MINUTES); return startTime; } private static TFramedTransport createScribeClient() throws TTransportException { final TSocket sock = new TSocket("127.0.0.1", 7911); final TFramedTransport transport = new TFramedTransport(sock); TBinaryProtocol protocol = new TBinaryProtocol(transport, false, false); client = new scribe.Client(protocol, protocol); transport.open(); return transport; } private static String createThriftPayload() throws TException { ArrayList<ThriftField> data = new ArrayList<ThriftField>(); data.add(ThriftField.createThriftField("Fuu", (short) 1)); data.add(ThriftField.createThriftField(true, (short) 2)); data.add(ThriftField.createThriftField(3.1459, (short) 3)); return String.format("%s:%s", new DateTime().getMillis(), new Base64().encodeToString(new ThriftFieldListSerializer().createPayload(data))); } @SuppressWarnings("unused") private static String createSmilePayload() throws IOException { final ByteArrayOutputStream stream = new ByteArrayOutputStream(); SmileFactory f = new SmileFactory(); JsonGenerator g = f.createJsonGenerator(stream); g.writeStartObject(); g.writeNumberField(SmileEnvelopeEvent.SMILE_EVENT_DATETIME_TOKEN_NAME, new DateTime().getMillis()); g.writeStringField("FuuField", "fuu"); g.writeBooleanField("TrueField", true); g.writeNumberField("Pi", 3.1459); g.writeNumberField("Long", 10001000000L); g.writeEndObject(); g.close(); // important: will force flushing of output, close underlying output stream return stream.toString(); } // private static int getPayloadSize() throws TException // { // TMemoryBuffer bufferTest = new TMemoryBuffer(32); // final TFramedTransport transportTest = new TFramedTransport(bufferTest); // TBinaryProtocol protocolTest = new TBinaryProtocol(transportTest, false, false); // messages.get(0).write(protocolTest); // transportTest.flush(); // return bufferTest.length(); // } }