com.ning.metrics.eventtracker.ScribeSender.java Source code

Java tutorial

Introduction

Here is the source code for com.ning.metrics.eventtracker.ScribeSender.java

Source

/*
 * Copyright 2010 Ning, Inc.
 *
 * Ning licenses this file to you under the Apache License, version 2.0
 * (the "License"); you may not use this file except in compliance with the
 * License.  You may obtain a copy of the License at:
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
 * License for the specific language governing permissions and limitations
 * under the License.
 */

package com.ning.metrics.eventtracker;

import com.ning.metrics.serialization.event.Event;
import com.ning.metrics.serialization.event.SmileBucketEvent;
import com.ning.metrics.serialization.util.Managed;
import com.ning.metrics.serialization.writer.CallbackHandler;
import org.apache.commons.codec.binary.Base64;
import org.apache.log4j.Logger;
import org.apache.thrift.transport.TTransportException;
import scribe.thrift.LogEntry;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.ObjectOutputStream;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;

/**
 * ScribeSender
 * <p/>
 * The class needs to be public for JMX.
 */
public class ScribeSender implements EventSender {
    private static final Logger log = Logger.getLogger(ScribeSender.class);

    private static final Charset CHARSET = Charset.forName("ISO-8859-1");

    private final AtomicInteger connectionRetries = new AtomicInteger(0);
    private ScribeClient scribeClient;

    private final AtomicInteger messagesSuccessfullySent = new AtomicInteger(0);
    private final AtomicInteger messagesSuccessfullySentSinceLastReconnection = new AtomicInteger(0);
    private int messagesToSendBeforeReconnecting = 0;

    private final AtomicBoolean sleeping = new AtomicBoolean(true);

    public ScribeSender(ScribeClient scribeClient, int messagesToSendBeforeReconnecting, int maxIdleTimeInMinutes) {
        this.scribeClient = scribeClient;
        this.messagesToSendBeforeReconnecting = messagesToSendBeforeReconnecting;

        // Setup a watchdog for the Scribe connection. We don't want to keep it open forever. For instance, SLB VIP
        // may trigger a RST if idle more than a few minutes.
        final ScheduledExecutorService executor = new ScheduledThreadPoolExecutor(1,
                Executors.defaultThreadFactory());
        executor.scheduleAtFixedRate(new Runnable() {
            @Override
            public void run() {
                if (sleeping.get()) {
                    log.info("Idle connection to Scribe, re-opening it");
                    createConnection();
                }
                sleeping.set(true);
            }
        }, maxIdleTimeInMinutes, maxIdleTimeInMinutes, TimeUnit.MINUTES);
    }

    /**
     * Re-initialize the connection with the Scribe endpoint.
     */
    public synchronized void createConnection() {
        if (scribeClient != null) {
            try {
                connectionRetries.incrementAndGet();
                scribeClient.closeLogger();
                scribeClient.openLogger();

                log.info("Connection to Scribe established");
            } catch (TTransportException e) {
                log.warn(String.format("Unable to connect to Scribe: %s", e.getLocalizedMessage()));
                scribeClient.closeLogger();
            }
        } else {
            log.warn("Scribe client has not been set up correctly.");
        }
    }

    /**
     * Disconnect from Scribe for good.
     */
    public void shutdown() {
        if (scribeClient != null) {
            scribeClient.closeLogger();
        }
    }

    @Override
    public void send(Event event, CallbackHandler handler) {
        if (scribeClient == null) {
            handler.onError(new Throwable("Scribe client has not been set up correctly"), event);
            return;
        }

        // Tell the watchdog that we are doing something
        sleeping.set(false);

        // TODO: offer batch API?, see drainEvents
        List<LogEntry> list = new ArrayList<LogEntry>(1);
        // TODO: update Scribe to pass a Thrift directly instead of serializing it

        final String logEntryMessage;
        try {
            logEntryMessage = eventToLogEntryMessage(event);
        } catch (IOException e) {
            handler.onError(new Throwable(e), event);
            return;
        }
        list.add(new LogEntry(event.getName(), logEntryMessage));

        try {
            scribeClient.log(list);
            messagesSuccessfullySent.addAndGet(list.size());
            messagesSuccessfullySentSinceLastReconnection.addAndGet(list.size());

            // For load balancing capabilities, we don't want to make sticky connections to Scribe.
            // After a certain threshold, force a refresh of the connection.
            if (messagesSuccessfullySentSinceLastReconnection.get() > messagesToSendBeforeReconnecting) {
                log.info("Recycling connection with Scribe");
                messagesSuccessfullySentSinceLastReconnection.set(0);
                createConnection();
            }
        } catch (org.apache.thrift.TException e) {
            // Connection flacky?
            log.warn(String.format("Error while sending message to Scribe: %s", e.getLocalizedMessage()));
            createConnection();
            handler.onError(new Throwable(e), event);
        }

        handler.onSuccess(event);
    }

    protected static String eventToLogEntryMessage(Event event) throws IOException {
        // Has the sender specified how to send the data?
        byte[] payload = event.getSerializedEvent();

        // Nope, default to ObjectOutputStream
        if (payload == null) {
            ByteArrayOutputStream out = new ByteArrayOutputStream();
            event.writeExternal(new ObjectOutputStream(out));
            payload = out.toByteArray();

            // 64-bit encode the serialized object
            payload = new Base64().encode(payload);
        }

        String scribePayload = new String(payload, CHARSET);

        // TODO Ugly code...
        if (event instanceof SmileBucketEvent) {
            return scribePayload;
        } else {
            // To avoid costly Thrift deserialization on the collector side, we embed the
            // timestamp in the format, outside of the payload. We need it for HDFS routing.
            return String.format("%s:%s", event.getEventDateTime().getMillis(), scribePayload);
        }
    }

    @Managed(description = "Get the number of messages successfully sent since startup to Scribe")
    public long getMessagesSuccessfullySent() {
        return messagesSuccessfullySent.get();
    }

    @Managed(description = "Get the number of messages successfully sent since last reconnection to Scribe")
    public long getMessagesSuccessfullySentSinceLastReconnection() {
        return messagesSuccessfullySentSinceLastReconnection.get();
    }

    @Managed(description = "Get the number of times we retried to connect to Scribe")
    public long getConnectionRetries() {
        return connectionRetries.get();
    }
}