com.ottogroup.bi.spqr.pipeline.queue.chronicle.DefaultStreamingMessageQueue.java Source code

Java tutorial

Introduction

Here is the source code for com.ottogroup.bi.spqr.pipeline.queue.chronicle.DefaultStreamingMessageQueue.java

Source

/**
 * Copyright 2015 Otto (GmbH & Co KG)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.ottogroup.bi.spqr.pipeline.queue.chronicle;

import java.io.File;
import java.io.IOException;
import java.util.Properties;
import java.util.concurrent.TimeUnit;

import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;

import com.codahale.metrics.Counter;
import com.ottogroup.bi.spqr.exception.RequiredInputMissingException;
import com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage;
import com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueue;
import com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueConsumer;
import com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueueProducer;
import com.ottogroup.bi.spqr.pipeline.queue.strategy.StreamingMessageQueueBlockingWaitStrategy;
import com.ottogroup.bi.spqr.pipeline.queue.strategy.StreamingMessageQueueDirectPassStrategy;
import com.ottogroup.bi.spqr.pipeline.queue.strategy.StreamingMessageQueueSleepingWaitStrategy;
import com.ottogroup.bi.spqr.pipeline.queue.strategy.StreamingMessageQueueWaitStrategy;

import net.openhft.chronicle.Chronicle;
import net.openhft.chronicle.ChronicleQueueBuilder;
import net.openhft.chronicle.VanillaChronicle;
import net.openhft.chronicle.tools.ChronicleTools;

/**
 * Implements a {@link StreamingMessageQueue} based on {@link Chronicle}
 * @author mnxfst
 * @since Mar 5, 2015
 */
public class DefaultStreamingMessageQueue implements StreamingMessageQueue {

    /** our faithful logging facility ... ;-) */
    private static final Logger logger = Logger.getLogger(DefaultStreamingMessageQueue.class);

    public static final String CFG_CHRONICLE_QUEUE_PATH = "queue.chronicle.path";
    public static final String CFG_CHRONICLE_QUEUE_DELETE_ON_EXIT = "queue.chronicle.deleteOnExist";
    public static final String CFG_CHRONICLE_QUEUE_ROLLING_INTERVAL = "queue.chronicle.rollingInterval";
    public static final String CFG_CHRONICLE_QUEUE_CYCLE_FORMAT = "queue.chronicle.cycleFormat";
    public static final String CFG_QUEUE_MESSAGE_WAIT_STRATEGY = "queue.message.waitStrategy";

    /** unique queue identifier */
    private String id = null;
    /** base path to use when creating/accessing the referenced chronicle files */
    private String basePath = null;
    /** delete the chronicle files on start and shutdown */
    private boolean deleteOnExit = true;
    /** chronicle file rolling interval - provided in minutes */
    private long queueRollingInterval = TimeUnit.MINUTES.toMillis(60);
    /** cycle format - default: yyyy-MM-dd/HH-mm */
    private String cycleFormat = "yyyy-MM-dd-HH-mm";
    /** chronicle instance - required for creating and accessing appender & tailer */
    private Chronicle chronicle = null;
    /** provides read access to chronicle */
    private DefaultStreamingMessageQueueConsumer queueConsumer = null;
    /** provides write access to chronicle */
    private DefaultStreamingMessageQueueProducer queueProducer = null;
    /** wait strategy applied on this queue */
    private StreamingMessageQueueWaitStrategy queueWaitStrategy = null;

    public long getSize() {
        return chronicle.size();
    }

    /**
     * @see com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueue#initialize(java.util.Properties)
     */
    public void initialize(Properties properties) throws RequiredInputMissingException {

        ////////////////////////////////////////////////////////////////////////////////
        // extract and validate input
        if (properties == null)
            throw new RequiredInputMissingException("Missing required properties");

        if (StringUtils.isBlank(this.id))
            throw new RequiredInputMissingException("Missing required queue identifier");

        if (StringUtils.equalsIgnoreCase(
                StringUtils.trim(properties.getProperty(CFG_CHRONICLE_QUEUE_DELETE_ON_EXIT)), "false"))
            this.deleteOnExit = false;

        this.basePath = StringUtils.lowerCase(StringUtils.trim(properties.getProperty(CFG_CHRONICLE_QUEUE_PATH)));
        if (StringUtils.isBlank(this.basePath))
            this.basePath = System.getProperty("java.io.tmpdir");

        String tmpCycleFormat = StringUtils
                .trim(properties.getProperty(CFG_CHRONICLE_QUEUE_CYCLE_FORMAT, "yyyy-MM-dd-HH-mm"));
        if (StringUtils.isNotBlank(tmpCycleFormat))
            this.cycleFormat = tmpCycleFormat;

        String pathToChronicle = this.basePath;
        if (!StringUtils.endsWith(pathToChronicle, File.separator))
            pathToChronicle = pathToChronicle + File.separator;
        pathToChronicle = pathToChronicle + id;

        try {
            this.queueRollingInterval = TimeUnit.MINUTES.toMillis(
                    Long.parseLong(StringUtils.trim(properties.getProperty(CFG_CHRONICLE_QUEUE_ROLLING_INTERVAL))));

            if (this.queueRollingInterval < VanillaChronicle.MIN_CYCLE_LENGTH) {
                this.queueRollingInterval = VanillaChronicle.MIN_CYCLE_LENGTH;
            }
        } catch (Exception e) {
            logger.info("Invalid queue rolling interval found: " + e.getMessage() + ". Using default: "
                    + TimeUnit.MINUTES.toMillis(60));
        }

        this.queueWaitStrategy = getWaitStrategy(
                StringUtils.trim(properties.getProperty(CFG_QUEUE_MESSAGE_WAIT_STRATEGY)));

        //
        ////////////////////////////////////////////////////////////////////////////////

        // clears the queue if requested 
        if (this.deleteOnExit) {
            ChronicleTools.deleteDirOnExit(pathToChronicle);
            ChronicleTools.deleteOnExit(pathToChronicle);
        }

        try {
            this.chronicle = ChronicleQueueBuilder.vanilla(pathToChronicle)
                    .cycleLength((int) this.queueRollingInterval).cycleFormat(this.cycleFormat).build();
            this.queueConsumer = new DefaultStreamingMessageQueueConsumer(this.getId(),
                    this.chronicle.createTailer(), this.queueWaitStrategy);
            this.queueProducer = new DefaultStreamingMessageQueueProducer(this.getId(),
                    this.chronicle.createAppender(), this.queueWaitStrategy);
        } catch (IOException e) {
            throw new RuntimeException(
                    "Failed to initialize chronicle at '" + pathToChronicle + "'. Error: " + e.getMessage());
        }

        logger.info("queue[type=chronicle, id=" + this.id + ", deleteOnExist=" + this.deleteOnExit + ", path="
                + pathToChronicle + "']");
    }

    /**
     * Return an instance of the referenced {@link StreamingMessageQueueWaitStrategy}
     * @param waitStrategyName name of strategy to instantiate (eg. {@link StreamingMessageQueueBlockingWaitStrategy#STRATEGY_NAME} (default))
     * @return
     */
    protected StreamingMessageQueueWaitStrategy getWaitStrategy(final String waitStrategyName) {
        if (StringUtils.equalsIgnoreCase(waitStrategyName, StreamingMessageQueueDirectPassStrategy.STRATEGY_NAME))
            return new StreamingMessageQueueDirectPassStrategy();
        else if (StringUtils.equalsIgnoreCase(waitStrategyName,
                StreamingMessageQueueSleepingWaitStrategy.STRATEGY_NAME))
            return new StreamingMessageQueueSleepingWaitStrategy();
        return new StreamingMessageQueueBlockingWaitStrategy();
    }

    /**
     * @see com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueue#shutdown()
     */
    public boolean shutdown() {
        try {
            this.chronicle.close();
            return true;
        } catch (IOException e) {
            logger.error("Failed to close underlying chronicle at " + basePath + "/" + id + ". Error: "
                    + e.getMessage());
            ;
        }
        return false;
    }

    /**
     * @see com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueue#insert(com.ottogroup.bi.spqr.pipeline.message.StreamingDataMessage)
     */
    public boolean insert(StreamingDataMessage message) {
        return queueProducer.insert(message); // TODO access appender directly
    }

    /**
     * @see com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueue#next()
     */
    public StreamingDataMessage next() {
        return queueConsumer.next(); // TODO access tailer directly
    }

    /**
     * @see com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueue#getProducer()
     */
    public StreamingMessageQueueProducer getProducer() {
        return this.queueProducer;
    }

    /**
     * @see com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueue#getConsumer()
     */
    public StreamingMessageQueueConsumer getConsumer() {
        return this.queueConsumer;
    }

    /**
     * @see com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueue#setId(java.lang.String)
     */
    public void setId(String id) {
        this.id = id;
    }

    /**
     * @see com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueue#getId()
     */
    public String getId() {
        return this.id;
    }

    /**
     * @see com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueue#setMessageInsertionCounter(com.codahale.metrics.Counter)
     */
    public void setMessageInsertionCounter(Counter counter) {
        this.queueProducer.setMessageInsertionCounter(counter);
    }

    /**
     * @see com.ottogroup.bi.spqr.pipeline.queue.StreamingMessageQueue#setMessageRetrievalCounter(com.codahale.metrics.Counter)
     */
    public void setMessageRetrievalCounter(Counter counter) {
        this.queueConsumer.setMessageRetrievalCounter(counter);
    }

}