com.erudika.para.queue.AWSQueueUtils.java Source code

Java tutorial

Introduction

Here is the source code for com.erudika.para.queue.AWSQueueUtils.java

Source

/*
 * Copyright 2013-2017 Erudika. https://erudika.com
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * For issues and patches go to: https://github.com/erudika
 */
package com.erudika.para.queue;

import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.regions.Region;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.sqs.AmazonSQSClient;
import com.amazonaws.services.sqs.model.CreateQueueRequest;
import com.amazonaws.services.sqs.model.DeleteMessageBatchRequestEntry;
import com.amazonaws.services.sqs.model.DeleteQueueRequest;
import com.amazonaws.services.sqs.model.Message;
import com.amazonaws.services.sqs.model.ReceiveMessageRequest;
import com.amazonaws.services.sqs.model.SendMessageBatchRequestEntry;
import com.erudika.para.DestroyListener;
import com.erudika.para.Para;
import com.erudika.para.annotations.Locked;
import com.erudika.para.core.ParaObject;
import com.erudika.para.core.Sysprop;
import com.erudika.para.core.Thing;
import com.erudika.para.core.utils.ParaObjectUtils;
import com.erudika.para.utils.Config;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Future;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Helper utilities for connecting to AWS SQS.
 * @author Alex Bogdanovski [alex@erudika.com]
 */
public final class AWSQueueUtils {

    private static AmazonSQSClient sqsClient;
    private static final int MAX_MESSAGES = 10; //max in bulk
    private static final int SLEEP = Config.getConfigInt("queue.polling_sleep_seconds", 60);
    private static final int POLLING_INTERVAL = Config.getConfigInt("queue.polling_interval_seconds",
            Config.IN_PRODUCTION ? 20 : 0);

    private static final String LOCAL_ENDPOINT = "http://localhost:9324";
    private static final Logger logger = LoggerFactory.getLogger(AWSQueueUtils.class);

    private static volatile Map<String, Future<?>> pollingThreads = new HashMap<String, Future<?>>();

    /**
     * No-args constructor.
     */
    private AWSQueueUtils() {
    }

    /**
     * Returns a client instance for AWS SQS.
     * @return a client that talks to SQS
     */
    public static AmazonSQSClient getClient() {
        if (sqsClient != null) {
            return sqsClient;
        }
        if (Config.IN_PRODUCTION) {
            Region region = Regions.getCurrentRegion();
            region = region != null ? region : Region.getRegion(Regions.fromName(Config.AWS_REGION));
            sqsClient = new AmazonSQSClient(new BasicAWSCredentials(Config.AWS_ACCESSKEY, Config.AWS_SECRETKEY))
                    .withRegion(region);
        } else {
            sqsClient = new AmazonSQSClient(new BasicAWSCredentials("x", "x")).withEndpoint(LOCAL_ENDPOINT);
        }

        Para.addDestroyListener(new DestroyListener() {
            public void onDestroy() {
                sqsClient.shutdown();
            }
        });
        return sqsClient;
    }

    /**
     * Creates a new SQS queue on AWS.
     * @param name queue name
     * @return the queue URL or null
     */
    public static String createQueue(String name) {
        if (StringUtils.isBlank(name)) {
            return null;
        }
        String queueURL = getQueueURL(name);
        if (queueURL == null) {
            try {
                queueURL = getClient().createQueue(new CreateQueueRequest(name)).getQueueUrl();
            } catch (AmazonServiceException ase) {
                logException(ase);
            } catch (AmazonClientException ace) {
                logger.error("Could not reach SQS. {0}", ace.toString());
            }
        }
        return queueURL;
    }

    /**
     * Deletes an SQS queue on AWS.
     * @param queueURL URL of the SQS queue
     */
    public static void deleteQueue(String queueURL) {
        if (!StringUtils.isBlank(queueURL)) {
            try {
                getClient().deleteQueue(new DeleteQueueRequest(queueURL));
            } catch (AmazonServiceException ase) {
                logException(ase);
            } catch (AmazonClientException ace) {
                logger.error("Could not reach SQS. {0}", ace.toString());
            }
        }
    }

    /**
     * Returns the SQS queue URL.
     * @param name queue name
     * @return the URL of the queue
     */
    public static String getQueueURL(String name) {
        try {
            return getClient().getQueueUrl(name).getQueueUrl();
        } catch (Exception e) {
            logger.info("Queue '{}' could not be found: {}", name, e.getMessage());
            return null;
        }
    }

    /**
     * Returns a list of URLs for all available queues on SQS.
     * @return a list or queue URLs
     */
    public static List<String> listQueues() {
        List<String> list = new ArrayList<String>();
        try {
            list = getClient().listQueues().getQueueUrls();
        } catch (AmazonServiceException ase) {
            logException(ase);
        } catch (AmazonClientException ace) {
            logger.error("Could not reach SQS. {0}", ace.toString());
        }
        return list;
    }

    /**
     * Pushes a number of messages in batch to an SQS queue.
     * @param queueURL the URL of the SQS queue
     * @param messages the massage bodies
     */
    public static void pushMessages(String queueURL, List<String> messages) {
        if (!StringUtils.isBlank(queueURL) && messages != null) {
            // only allow strings - ie JSON
            try {
                int j = 0;
                List<SendMessageBatchRequestEntry> msgs = new ArrayList<SendMessageBatchRequestEntry>(MAX_MESSAGES);
                for (int i = 0; i < messages.size(); i++) {
                    String message = messages.get(i);
                    if (!StringUtils.isBlank(message)) {
                        msgs.add(new SendMessageBatchRequestEntry().withMessageBody(message).withId(i + ""));
                    }
                    if (++j >= MAX_MESSAGES || i == messages.size() - 1) {
                        if (!msgs.isEmpty()) {
                            getClient().sendMessageBatch(queueURL, msgs);
                            msgs.clear();
                        }
                        j = 0;
                    }
                }
            } catch (AmazonServiceException ase) {
                logException(ase);
            } catch (AmazonClientException ace) {
                logger.error("Could not reach SQS. {}", ace.toString());
            }
        }
    }

    /**
     * Pulls a number of messages from an SQS queue.
     * @param queueURL the URL of the SQS queue
     * @param numberOfMessages the number of messages to pull
     * @return a list of messages
     */
    public static List<String> pullMessages(String queueURL, int numberOfMessages) {
        List<String> messages = new ArrayList<String>();
        if (!StringUtils.isBlank(queueURL)) {
            try {
                int batchSteps = 1;
                int maxForBatch = numberOfMessages;
                if ((numberOfMessages > MAX_MESSAGES)) {
                    batchSteps = (numberOfMessages / MAX_MESSAGES)
                            + ((numberOfMessages % MAX_MESSAGES > 0) ? 1 : 0);
                    maxForBatch = MAX_MESSAGES;
                }

                for (int i = 0; i < batchSteps; i++) {
                    List<Message> list = getClient().receiveMessage(new ReceiveMessageRequest(queueURL)
                            .withMaxNumberOfMessages(maxForBatch).withWaitTimeSeconds(POLLING_INTERVAL))
                            .getMessages();
                    if (list != null && !list.isEmpty()) {
                        List<DeleteMessageBatchRequestEntry> del = new ArrayList<DeleteMessageBatchRequestEntry>();
                        for (Message msg : list) {
                            messages.add(msg.getBody());
                            del.add(new DeleteMessageBatchRequestEntry(msg.getMessageId(), msg.getReceiptHandle()));
                        }
                        getClient().deleteMessageBatch(queueURL, del);
                    }
                }
            } catch (AmazonServiceException ase) {
                logException(ase);
            } catch (AmazonClientException ace) {
                logger.error("Could not reach SQS. {}", ace.toString());
            }
        }
        return messages;
    }

    /**
     * Starts polling for messages from SQS in a separate thread.
     * @param queueURL a queue URL
     */
    public static void startPollingForMessages(final String queueURL) {
        if (!StringUtils.isBlank(queueURL) && !pollingThreads.containsKey(queueURL)) {
            logger.info("Starting SQS river using queue {} (polling interval: {}s)", queueURL, POLLING_INTERVAL);
            pollingThreads.put(queueURL, Para.getExecutorService().submit(new SQSRiver(queueURL)));
            Para.addDestroyListener(new DestroyListener() {
                public void onDestroy() {
                    stopPollingForMessages(queueURL);
                }
            });
        }
    }

    /**
     * Stops the thread that has been polling for messages.
     * @param queueURL the queue URL
     */
    public static void stopPollingForMessages(String queueURL) {
        if (!StringUtils.isBlank(queueURL) && pollingThreads.containsKey(queueURL)) {
            logger.info("Stopping SQS river on queue {} ...", queueURL);
            pollingThreads.get(queueURL).cancel(true);
            pollingThreads.remove(queueURL);
        }
    }

    /**
     * An SQS river.
     * Adapted from https://github.com/albogdano/elasticsearch-river-amazonsqs
     */
    static class SQSRiver implements Runnable {

        private int idleCount = 0;
        private final String queueURL;

        SQSRiver(String queueURL) {
            this.queueURL = queueURL;
        }

        @SuppressWarnings("unchecked")
        public void run() {
            ArrayList<ParaObject> createList = new ArrayList<ParaObject>();
            ArrayList<ParaObject> updateList = new ArrayList<ParaObject>();
            ArrayList<ParaObject> deleteList = new ArrayList<ParaObject>();

            while (true) {
                logger.debug("Waiting {}s for messages...", POLLING_INTERVAL);
                List<String> msgs = pullMessages(queueURL, MAX_MESSAGES);
                logger.debug("Pulled {} messages from queue.", msgs.size());

                try {
                    for (final String msg : msgs) {
                        logger.debug("SQS MESSAGE: {}", msg);
                        if (StringUtils.contains(msg, Config._APPID) && StringUtils.contains(msg, Config._TYPE)) {
                            parseAndCategorizeMessage(msg, createList, updateList, deleteList);
                        }
                    }

                    if (!createList.isEmpty() || !updateList.isEmpty() || !deleteList.isEmpty()) {
                        Para.getDAO().createAll(createList);
                        Para.getDAO().updateAll(updateList);
                        Para.getDAO().deleteAll(deleteList);
                        logger.debug("Objects pulled from SQS queue: {} created, {} updated, {} deleted.",
                                createList.size(), updateList.size(), deleteList.size());
                        createList.clear();
                        updateList.clear();
                        deleteList.clear();
                        idleCount = 0;
                    } else if (msgs.isEmpty()) {
                        idleCount++;
                        // no tasks in queue => throttle down pull requests
                        if (SLEEP > 0 && idleCount >= 3) {
                            try {
                                logger.debug("Queue {} is empty. Sleeping for {}s...", queueURL, SLEEP);
                                Thread.sleep(SLEEP * 1000);
                            } catch (InterruptedException e) {
                                logger.info("SQS river interrupted.");
                                break;
                            }
                        }
                    }
                } catch (Exception e) {
                    logger.error("Batch processing operation failed: {}", e);
                }
            }
        }
    }

    private static void parseAndCategorizeMessage(final String msg, ArrayList<ParaObject> createList,
            ArrayList<ParaObject> updateList, ArrayList<ParaObject> deleteList) throws IOException {
        Map<String, Object> parsed = ParaObjectUtils.getJsonReader(Map.class).readValue(msg);
        String id = parsed.containsKey(Config._ID) ? (String) parsed.get(Config._ID) : null;
        String type = (String) parsed.get(Config._TYPE);
        String appid = (String) parsed.get(Config._APPID);
        Class<?> clazz = ParaObjectUtils.toClass(type);
        boolean isWhitelistedType = clazz.equals(Thing.class) || clazz.equals(Sysprop.class);

        if (!StringUtils.isBlank(appid) && isWhitelistedType) {
            if (parsed.containsKey("_delete") && "true".equals(parsed.get("_delete"))) {
                Sysprop s = new Sysprop(id);
                s.setAppid(appid);
                deleteList.add(s);
            } else {
                if (id == null) {
                    ParaObject obj = ParaObjectUtils.setAnnotatedFields(parsed);
                    if (obj != null) {
                        createList.add(obj);
                    }
                } else {
                    updateList.add(ParaObjectUtils.setAnnotatedFields(Para.getDAO().read(appid, id), parsed,
                            Locked.class));
                }
            }
        }
    }

    private static void logException(AmazonServiceException ase) {
        logger.error("AmazonServiceException: error={}, statuscode={}, awserrcode={}, errtype={}, reqid={}",
                ase.toString(), ase.getStatusCode(), ase.getErrorCode(), ase.getErrorType(), ase.getRequestId());
    }

}