com.ottogroup.bi.asap.operator.webtrends.consumer.WebtrendsStreamsConsumer.java Source code

Java tutorial

Introduction

Here is the source code for com.ottogroup.bi.asap.operator.webtrends.consumer.WebtrendsStreamsConsumer.java

Source

/**
 * Copyright 2014 Otto (GmbH & Co KG)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.ottogroup.bi.asap.operator.webtrends.consumer;

import java.io.IOException;
import java.net.URI;
import java.util.Properties;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;

import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.eclipse.jetty.websocket.api.Session;
import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose;
import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect;
import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage;
import org.eclipse.jetty.websocket.api.annotations.WebSocket;
import org.eclipse.jetty.websocket.client.ClientUpgradeRequest;
import org.eclipse.jetty.websocket.client.WebSocketClient;

import com.ottogroup.bi.asap.component.ComponentType;
import com.ottogroup.bi.asap.component.annotation.AsapComponent;
import com.ottogroup.bi.asap.component.source.Source;
import com.ottogroup.bi.asap.exception.RequiredInputMissingException;
import com.ottogroup.bi.asap.mailbox.Mailbox;
import com.ottogroup.bi.asap.message.StreamingDataMessage;

/**
 * Establishes and maintains a connection with streams.webtrends.com where it reads 
 * click-stream data from and emits it to {@link Pipeline} it is associated with
 * @author mnxfst
 * @since Nov 30, 2014
 */
@AsapComponent(type = ComponentType.SOURCE, name = "webtrendsStreamsConsumer", version = "0.0.1", description = "Consumes the webtrends streams api")
@WebSocket
public class WebtrendsStreamsConsumer implements Source {

    /** our faithful logging service */
    private static final Logger logger = Logger.getLogger(WebtrendsStreamsConsumer.class);

    ////////////////////////////////////////////////////////////////////
    // required config options
    public static final String CFG_COMPONENT_ID = "componentId";
    public static final String CFG_WT_AUTH_AUDIENCE = "webtrends.auth.audience";
    public static final String CFG_WT_AUTH_SCOPE = "webtrends.auth.scope";
    public static final String CFG_WT_AUTH_URL = "webtrends.auth.url";
    public static final String CFG_WT_CLIENT_ID = "webtrends.client.id";
    public static final String CFG_WT_CLIENT_SECRET = "webtrends.client.secret";
    public static final String CFG_WT_STREAM_URL = "webtrends.stream.url";
    public static final String CFG_WT_STREAM_TYPE = "webtrends.stream.type";
    public static final String CFG_WT_STREAM_QUERY = "webtrends.stream.query";
    public static final String CFG_WT_STREAM_VERSION = "webtrends.stream.version";
    public static final String CFG_WT_SCHEMA_VERSION = "webtrends.schema.version";
    //
    ////////////////////////////////////////////////////////////////////

    ////////////////////////////////////////////////////////////////////
    // settings required to establish a connection with webtrends streams api
    private String authUrl;
    private String authAudience;
    private String authScope;
    private String clientId;
    private String clientSecret;
    private String eventStreamUrl;
    private String streamType;
    private String streamQuery;
    private String streamVersion;
    private String schemaVersion;

    /** identifier of this component instance */
    private String componentId = null;
    /** number of messages processed by this instance */
    private long numProcessedMessages = 0;
    /** mailbox */
    private Mailbox mailbox;
    /** required for timeout handling when connecting with api */
    private final CountDownLatch latch = new CountDownLatch(1);
    /** OAuth token received from webtrends */
    private String oAuthToken;
    /** client used to establish and maintain the websocket connection */
    private WebSocketClient webtrendsStreamSocketClient;
    /** associated websocket session */
    private Session websocketSession;
    /** internal message queue used for buffering before data is being handed over to publisher */
    private final BlockingQueue<String> streamMessageQueue = new LinkedBlockingQueue<String>(100000);
    /** run state */
    private boolean isRunning = false;

    /**
     * @see com.ottogroup.bi.asap.component.Component#init(java.util.Properties)
     */
    public void init(Properties properties) throws RequiredInputMissingException {

        //////////////////////////////////////////////////////////////////
        // extract settings and validate values
        this.authAudience = properties.getProperty(CFG_WT_AUTH_AUDIENCE);
        this.authScope = properties.getProperty(CFG_WT_AUTH_SCOPE);
        this.authUrl = properties.getProperty(CFG_WT_AUTH_URL);
        this.clientId = properties.getProperty(CFG_WT_CLIENT_ID);
        this.clientSecret = properties.getProperty(CFG_WT_CLIENT_SECRET);
        this.eventStreamUrl = properties.getProperty(CFG_WT_STREAM_URL);
        this.streamType = properties.getProperty(CFG_WT_STREAM_TYPE);
        this.streamQuery = properties.getProperty(CFG_WT_STREAM_QUERY);
        this.streamVersion = properties.getProperty(CFG_WT_STREAM_VERSION);
        this.schemaVersion = properties.getProperty(CFG_WT_SCHEMA_VERSION);
        this.componentId = properties.getProperty(CFG_COMPONENT_ID);

        if (StringUtils.isBlank(authAudience))
            throw new RequiredInputMissingException(
                    "Missing required input for parameter '" + CFG_WT_AUTH_AUDIENCE + "'");
        if (StringUtils.isBlank(authScope))
            throw new RequiredInputMissingException(
                    "Missing required input for parameter '" + CFG_WT_AUTH_SCOPE + "'");
        if (StringUtils.isBlank(authUrl))
            throw new RequiredInputMissingException(
                    "Missing required input for parameter '" + CFG_WT_AUTH_URL + "'");
        if (StringUtils.isBlank(clientId))
            throw new RequiredInputMissingException(
                    "Missing required input for parameter '" + CFG_WT_CLIENT_ID + "'");
        if (StringUtils.isBlank(clientSecret))
            throw new RequiredInputMissingException(
                    "Missing required input for parameter '" + CFG_WT_CLIENT_SECRET + "'");
        if (StringUtils.isBlank(eventStreamUrl))
            throw new RequiredInputMissingException(
                    "Missing required input for parameter '" + CFG_WT_STREAM_URL + "'");
        if (StringUtils.isBlank(streamType))
            throw new RequiredInputMissingException(
                    "Missing required input for parameter '" + CFG_WT_STREAM_TYPE + "'");
        if (StringUtils.isBlank(streamQuery))
            throw new RequiredInputMissingException(
                    "Missing required input for parameter '" + CFG_WT_STREAM_QUERY + "'");
        if (StringUtils.isBlank(streamVersion))
            throw new RequiredInputMissingException(
                    "Missing required input for parameter '" + CFG_WT_STREAM_VERSION + "'");
        if (StringUtils.isBlank(schemaVersion))
            throw new RequiredInputMissingException(
                    "Missing required input for parameter '" + CFG_WT_SCHEMA_VERSION + "'");
        if (StringUtils.isBlank(componentId))
            throw new RequiredInputMissingException(
                    "Missing required input for parameter '" + CFG_COMPONENT_ID + "'");
        //
        //////////////////////////////////////////////////////////////////

        // authenticate with the webtrends service
        WebtrendsTokenRequest tokenRequest = new WebtrendsTokenRequest(this.authUrl, this.authAudience,
                this.authScope, this.clientId, this.clientSecret);
        try {
            this.oAuthToken = tokenRequest.execute();
        } catch (Exception e) {
            throw new RuntimeException("Failed to request token from '" + authUrl + "'. Error: " + e.getMessage());
        }

        // initialize the webtrends stream socket client and connect the listener
        this.webtrendsStreamSocketClient = new WebSocketClient();
        try {
            this.webtrendsStreamSocketClient.start();
            ClientUpgradeRequest upgradeRequest = new ClientUpgradeRequest();
            this.webtrendsStreamSocketClient.connect(this, new URI(this.eventStreamUrl), upgradeRequest);
            await(5, TimeUnit.SECONDS);
        } catch (Exception e) {
            throw new RuntimeException("Unable to connect to web socket: " + e.getMessage(), e);
        }

        this.isRunning = true;
    }

    /**
     * @see com.ottogroup.bi.asap.node.pipeline.component.DataComponent#shutdown()
     */
    public boolean shutdown() {
        try {
            this.websocketSession.close();
        } catch (Exception e) {
            logger.error("Failed to close websocket session: " + e.getMessage());
        }
        try {
            this.webtrendsStreamSocketClient.stop();
        } catch (Exception e) {
            logger.error("Failed to close websocket client: " + e.getMessage());
        }
        return true;
    }

    /**
     * Executed after establishing web socket connection with streams api
     * @param session
     */
    @OnWebSocketConnect
    public void onConnect(Session session) {
        this.websocketSession = session;
        sendUpdate(this.websocketSession, this.oAuthToken, this.streamType, this.streamQuery, this.streamVersion,
                this.schemaVersion);
    }

    /**
     * Executed by web socket implementation when receiving a message from the
     * streams api. The message will be directly handed over to the configured 
     * {@link ActorRef message receiver}  
     * @param message
     */
    @OnWebSocketMessage
    public void onMessage(String message) {
        try {
            this.streamMessageQueue.offer(message, 1000, TimeUnit.MILLISECONDS);
        } catch (InterruptedException e) {
            logger.error("Failed to offer element to internal queue. Ignoring event. Error: " + e.getMessage());
        }
    }

    /**
     * Executed when closing the web socket connection
     * @param statusCode
     * @param reason
     */
    @OnWebSocketClose
    public void onClose(int statusCode, String reason) {
        //
    }

    /**
     * @see java.lang.Runnable#run()
     */
    public void run() {
        if (logger.isDebugEnabled())
            logger.debug("twitter stream consumer initialized [id=" + componentId + "]");

        // keep on consuming until either the consumer or the client is interrupted  
        while (this.isRunning && this.websocketSession.isOpen()) {
            try {
                String msg = streamMessageQueue.poll(100, TimeUnit.MILLISECONDS);
                if (msg != null) {
                    this.mailbox
                            .insert(new StreamingDataMessage(this.componentId, msg, System.currentTimeMillis()));
                    // TODO back pressure handling
                    this.numProcessedMessages++;
                }

            } catch (InterruptedException e) {
                logger.error("Failed to read data from websocket. Error: " + e.getMessage());
            }
        }

        shutdown();

        logger.info("webtrends stream consumer received " + this.numProcessedMessages + " messages");

    }

    /**
     * Timeout handler
     * @param duration
     * @param unit
     * @return
     * @throws InterruptedException
     */
    public boolean await(int duration, TimeUnit unit) throws InterruptedException {
        return latch.await(duration, unit);
    }

    /**
     * Sends an update towards the webtrends stream api using the contents of the 
     * provided {@link WebtrendsStreamListenerQueryUpdateMessage message}
     * @param msg
     */
    protected void sendUpdate(final Session session, final String oAuthToken, final String streamType,
            final String streamQuery, final String streamVersion, final String schemaVersion) {

        // build SAPI query object
        final StringBuilder sb = new StringBuilder();
        sb.append("{\"access_token\":\"");
        sb.append(oAuthToken);
        sb.append("\",\"command\":\"stream\"");
        sb.append(",\"stream_type\":\"");
        sb.append(streamType);
        sb.append("\",\"query\":\"");
        sb.append(streamQuery);
        sb.append("\",\"api_version\":\"");
        sb.append(streamVersion);
        sb.append("\",\"schema_version\":\"");
        sb.append(schemaVersion);
        sb.append("\"}");

        try {
            session.getRemote().sendString(sb.toString());
        } catch (IOException e) {
            throw new RuntimeException("Unable to open stream", e);
        }
    }

    /**
     * @see com.ottogroup.bi.asap.component.Component#getId()
     */
    public String getId() {
        return this.componentId;
    }

    /**
     * @see com.ottogroup.bi.asap.component.Component#setId(java.lang.String)
     */
    public void setId(String id) {
        this.componentId = id;
    }

    /**
     * @see com.ottogroup.bi.asap.component.Component#getTotalNumOfMessages()
     */
    public long getTotalNumOfMessages() {
        return this.numProcessedMessages;
    }

    /**
     * @see com.ottogroup.bi.asap.component.source.Source#setMailbox(com.ottogroup.bi.asap.mailbox.Mailbox)
     */
    public void setMailbox(Mailbox mailbox) {
        this.mailbox = mailbox;
    }

}