org.opencommercesearch.feed.BaseRestFeed.java Source code

Java tutorial

Introduction

Here is the source code for org.opencommercesearch.feed.BaseRestFeed.java

Source

package org.opencommercesearch.feed;

/*
* Licensed to OpenCommerceSearch under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. OpenCommerceSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import atg.beans.DynamicBeans;
import atg.beans.DynamicPropertyDescriptor;
import atg.beans.PropertyNotFoundException;
import atg.json.JSONArray;
import atg.json.JSONException;
import atg.json.JSONObject;
import atg.nucleus.GenericService;
import atg.nucleus.ServiceException;
import atg.repository.*;
import atg.repository.rql.RqlStatement;
import org.apache.commons.lang.StringUtils;
import org.opencommercesearch.api.ProductService;
import org.restlet.Request;
import org.restlet.Response;
import org.restlet.data.Encoding;
import org.restlet.data.MediaType;
import org.restlet.data.Method;
import org.restlet.data.Status;
import org.restlet.engine.application.EncodeRepresentation;
import org.restlet.representation.Representation;
import org.restlet.representation.StreamRepresentation;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;

import static org.opencommercesearch.api.ProductService.Endpoint;

/**
 * Base feed class that sends repository items to the OCS REST API.
 * <p/>
 * Data is transferred in JSON format and through regular HTTP calls. Child classes should handle
 * repository item to JSON transforms.
 * <p/>
 * Notice that the API must be aware of the repository item being fed (checkout opencommercesearch-api project).
 * <p/>
 * A feed can be transactional or not. When using transactions, all items will be cleared first. If an exception happens
 * all changes will be rolled back. If not, changes are committed. Be aware when using Solr. Solr transactions are not isolated.
 * Make sure the SolrCore doesn't have auto commits enabled.
 *
 * Non transactional feeds will delete all items that were not updated during the feed.
 *
 * @author jmendez
 */
public abstract class BaseRestFeed extends GenericService {

    /**
     * Repository to query for items
     */
    private Repository repository;

    /**
     * The actual item descriptor name being fed.
     */
    private String itemDescriptorName;

    /**
     * RQL query to get total items to be fed
     */
    private RqlStatement countRql;

    /**
     * Actual RQL that will get all the items to be fed (and fields for each item)
     */
    private RqlStatement rql;

    /**
     * Number of items that will be sent on each request
     */
    private int batchSize;

    /**
     * Whether or not this feed is enabled.
     */
    private boolean enabled;

    /**
     * Max error percentage tolerated by this feed. If this threshold is reached, then the feed will be discarded
     * since it will be considered risky. I.e. set it to 0.1 if you want a maximum of 10% errors of the total items
     * cause the feed to stop.
     */
    private double errorThreshold;

    /**
     * Whether or not this feeds is transactional.
     */
    private boolean transactional = true;

    private ProductService productService;

    private String endpointUrl;

    private Map<String, String> customPropertyMappings;

    /**
     * The custom property mappings loaded based on customPropertyMappings. For example
     * category.shortDisplayName -> alias becomes category -> (shortDisplayName -> alias)
     */
    private Map<String, Map<String, String>> itemDescriptorCustomPropertyMappings;

    public Repository getRepository() {
        return repository;
    }

    public void setRepository(Repository repository) {
        this.repository = repository;
    }

    public String getItemDescriptorName() {
        return itemDescriptorName;
    }

    public void setItemDescriptorName(String itemDescriptorName) {
        this.itemDescriptorName = itemDescriptorName;
    }

    public RqlStatement getCountRql() {
        return countRql;
    }

    public void setCountRql(RqlStatement countRql) {
        this.countRql = countRql;
    }

    public RqlStatement getRql() {
        return rql;
    }

    public void setRql(RqlStatement rql) {
        this.rql = rql;
    }

    public int getBatchSize() {
        return batchSize;
    }

    public void setBatchSize(int batchSize) {
        this.batchSize = batchSize;
    }

    public boolean isEnabled() {
        return enabled;
    }

    public void setTransactional(boolean transactional) {
        this.transactional = transactional;
    }

    public boolean isTransactional() {
        return transactional;
    }

    public void setEnabled(boolean enabled) {
        this.enabled = enabled;
    }

    public ProductService getProductService() {
        return productService;
    }

    public void setProductService(ProductService productService) {
        this.productService = productService;
    }

    public double getErrorThreshold() {
        return errorThreshold;
    }

    public void setErrorThreshold(double errorThreshold) {
        this.errorThreshold = errorThreshold;
    }

    @Override
    public void doStartService() throws ServiceException {
        if (getProductService() == null) {
            throw new ServiceException("No productService found");
        }
        endpointUrl = getProductService().getUrl4Endpoint(getEndpoint());

        if (getCustomPropertyMappings() != null && getCustomPropertyMappings().size() > 0) {
            initializeCustomPropertyMappings();
        }
    }

    /**
     * Helper method to initialize the custom properties for each item descriptor
     */
    private void initializeCustomPropertyMappings() {
        try {
            itemDescriptorCustomPropertyMappings = new HashMap<String, Map<String, String>>();
            for (Map.Entry<String, String> entry : getCustomPropertyMappings().entrySet()) {
                String[] parts = StringUtils.split(entry.getKey(), ".", 2);

                if (parts.length == 2) {
                    String itemDescriptorName = parts[0];
                    String propertyName = parts[1];
                    String propertyAlias = entry.getValue();

                    processMapping(itemDescriptorName, propertyName, propertyAlias);
                } else {
                    throw new IllegalArgumentException("Invalid nested property '" + entry.getKey() + "'");
                }
            }
        } catch (RepositoryException ex) {
            if (isLoggingError()) {
                logDebug("Cannot load category descriptor", ex);
            }
        }
    }

    /**
     * Helper method to process a mapping for an item descriptor
     *
     * @param itemDescriptorName is the item descriptor name
     * @param propertyName is the property name in the item descriptor
     * @param propertyAlias is the property alias to be used in the JSON objects
     */
    private void processMapping(String itemDescriptorName, String propertyName, String propertyAlias)
            throws RepositoryException {
        RepositoryItemDescriptor itemDescriptor = getRepository().getItemDescriptor(itemDescriptorName);

        if (itemDescriptor != null) {
            RepositoryPropertyDescriptor propertyDescriptor = getPropertyDescriptor(itemDescriptor, propertyName);

            if (propertyDescriptor != null) {
                Map<String, String> mappings = itemDescriptorCustomPropertyMappings.get(itemDescriptorName);

                if (mappings == null) {
                    mappings = new LinkedHashMap<String, String>();
                    itemDescriptorCustomPropertyMappings.put(itemDescriptorName, mappings);
                }
                mappings.put(propertyName, propertyAlias);
            } else {
                if (isLoggingError()) {
                    logError("Property descriptor not found for '" + propertyName + "' in item descriptor '"
                            + itemDescriptor.getItemDescriptorName() + "'");
                }
            }
        } else {
            if (isLoggingError()) {
                logError("Item descriptor not found '" + itemDescriptorName + "'");
            }
        }
    }

    private RepositoryPropertyDescriptor getPropertyDescriptor(RepositoryItemDescriptor itemDescriptor,
            String propertyName) {
        int index = propertyName.indexOf(".");

        if (index == -1) {
            return asRepositoryPropertyDescriptor(itemDescriptor.getPropertyDescriptor(propertyName),
                    itemDescriptor);
        } else {
            String property = propertyName.substring(0, index);
            String nestedProperty = propertyName.substring(index + 1);
            RepositoryPropertyDescriptor propertyDescriptor = asRepositoryPropertyDescriptor(
                    itemDescriptor.getPropertyDescriptor(property), itemDescriptor);

            if (propertyDescriptor == null) {
                return null;
            }
            return getPropertyDescriptor(propertyDescriptor.getPropertyItemDescriptor(), nestedProperty);
        }
    }

    private RepositoryPropertyDescriptor asRepositoryPropertyDescriptor(
            DynamicPropertyDescriptor propertyDescriptor, RepositoryItemDescriptor itemDescriptor) {
        if (propertyDescriptor instanceof RepositoryItemDescriptor) {
            if (isLoggingError()) {
                logError("Only RepositoryPropertyDescriptor is supported, skipping " + propertyDescriptor.getName()
                        + " on " + itemDescriptor);
            }
            return null;
        }
        return (RepositoryPropertyDescriptor) propertyDescriptor;
    }

    /**
     * Helper method to populate custom properties from a repository item into a jsonObject
     */
    protected void setCustomProperties(RepositoryItem item, JSONObject json)
            throws RepositoryException, JSONException {
        if (itemDescriptorCustomPropertyMappings == null || itemDescriptorCustomPropertyMappings.size() == 0) {
            return;
        }

        String itemDescriptorName = item.getItemDescriptor().getItemDescriptorName();
        Map<String, String> mappings = itemDescriptorCustomPropertyMappings.get(itemDescriptorName);
        if (mappings != null) {
            for (Map.Entry<String, String> entry : mappings.entrySet()) {
                String jsonPropertyName = entry.getValue();
                try {
                    Object propertyValue = DynamicBeans.getSubPropertyValue(item, entry.getKey());
                    if (jsonPropertyName.startsWith("attributes.")) {
                        if (!json.has("attributes")) {
                            json.put("attributes", new JSONObject());
                        }
                        JSONObject attributes = json.getJSONObject("attributes");
                        if (propertyValue != null) {
                            attributes.put(jsonPropertyName.substring("attributes.".length()),
                                    propertyValue.toString());
                        }
                    } else {
                        json.put(jsonPropertyName, propertyValue);
                    }
                    if (isLoggingDebug()) {
                        logDebug("Setting '" + jsonPropertyName + "' to '" + propertyValue + "' for " + item);
                    }
                } catch (PropertyNotFoundException ex) {
                    if (isLoggingError()) {
                        logError("Property " + entry.getKey() + " not found in " + item, ex);
                    }
                }
            }
        }
    }

    /**
     * Start running this feed.
     * @throws RepositoryException If there are problems reading the repository items from the database.
     */
    public void startFeed() throws RepositoryException, IOException {
        if (!isEnabled()) {
            if (isLoggingInfo()) {
                logInfo("Did not start feed for " + itemDescriptorName
                        + " since is disabled. Verify your configuration is correct.");
            }

            return;
        }

        long startTime = System.currentTimeMillis();
        int processed = 0;
        int failed = 0;

        RepositoryView itemView = getRepository().getView(itemDescriptorName);
        int count = countRql.executeCountQuery(itemView, null);
        int errorThreshold = getErrorThreshold() <= Double.MIN_VALUE ? count
                : (int) Math.ceil((count * getErrorThreshold()));

        if (isLoggingInfo()) {
            logInfo("Started " + itemDescriptorName + " feed for " + count + " items.");
        }

        try {
            long feedTimestamp = System.currentTimeMillis();

            if (count > 0) {
                if (isTransactional()) {
                    sendDeleteByQuery();
                }

                Integer[] rqlArgs = new Integer[] { 0, getBatchSize() };
                RepositoryItem[] items = rql.executeQueryUncached(itemView, rqlArgs);

                while (items != null) {
                    try {
                        int sent = sendItems(items, feedTimestamp);
                        processed += sent;
                        failed += items.length - sent;
                    } catch (Exception ex) {

                        if (isLoggingError()) {
                            logError("Cannot send " + itemDescriptorName + "[" + getIdsFromItemsArray(items) + "]",
                                    ex);
                        }

                        // Retry one by one
                        if (isLoggingInfo()) {
                            logInfo("Retrying batch one by one");
                        }

                        for (RepositoryItem item : items) {
                            if (item == null) {
                                continue;
                            }
                            RepositoryItem[] onlyOneItem = new RepositoryItem[] { item };
                            try {
                                int sent = sendItems(onlyOneItem, feedTimestamp);
                                processed += sent;
                                failed += onlyOneItem.length - sent;
                            } catch (Exception ex2) {
                                failed++;
                                if (isLoggingError()) {
                                    logError("Cannot resend " + itemDescriptorName + "["
                                            + getIdsFromItemsArray(onlyOneItem) + "]", ex2);
                                }
                            }
                        }
                    }

                    if (isLoggingInfo()) {
                        logInfo("Processed " + processed + " " + itemDescriptorName + " items out of " + count
                                + " with " + failed + " failures");
                    }

                    if (failed < errorThreshold) {
                        //Get the next batch only if the feed is performing well.
                        rqlArgs[0] += getBatchSize();
                        items = rql.executeQueryUncached(itemView, rqlArgs);
                    } else {
                        //Error threshold reached. Stop.
                        break;
                    }
                }

                if (failed < errorThreshold) {
                    //Send commit or deletes if the feeds looks healthy.
                    if (isTransactional()) {
                        sendCommit();
                    } else {
                        sendDelete(feedTimestamp);
                    }
                } else {
                    if (isLoggingError()) {
                        logError(itemDescriptorName
                                + " feed interrupted since it seems to be failing too often. At least "
                                + (getErrorThreshold() * 100) + "% out of " + count + " items had errors");
                    }

                    if (isTransactional()) {
                        //Roll back as much as we can from the changes done before the threshold was reached (specially initial delete)
                        sendRollback();
                    }
                }
            } else {
                if (isLoggingInfo()) {
                    logInfo("No " + itemDescriptorName + " items found. Nothing to do here.");
                }
            }
        } catch (Exception e) {
            if (isLoggingError()) {
                logError("Error while processing feed.", e);
            }

            if (isTransactional()) {
                sendRollback();
            }
        }

        if (isLoggingInfo()) {
            logInfo(itemDescriptorName + " feed finished in " + ((System.currentTimeMillis() - startTime) / 1000)
                    + " seconds, " + processed + " processed items and " + failed + " failures.");
        }
    }

    /**
     * Convert the given items to a JSON list and post them to the given API endpoint.
     * @param itemList The list of repository items to be sent.
     * @param feedTimestamp is the feed timestamp
     * @return The total count of items sent.
     * @throws RepositoryException if item data from the list can't be read.
     */
    private int sendItems(RepositoryItem[] itemList, long feedTimestamp) throws RepositoryException {
        int sent = 0;

        try {
            final JSONArray jsonObjects = new JSONArray();

            for (RepositoryItem item : itemList) {
                if (item == null) {
                    continue;
                }
                JSONObject json = repositoryItemToJson(item);
                setCustomProperties(item, json);

                if (json == null) {
                    if (isLoggingDebug()) {
                        logDebug("Sending " + itemDescriptorName + "[" + item.getRepositoryId()
                                + "] failed because it is missing required information. Expected: "
                                + Arrays.toString(getRequiredItemFields()));
                    }
                } else {
                    jsonObjects.add(json);
                    sent++;
                }
            }

            //If all items were ignored due data errors, simply ignore this batch.
            if (jsonObjects.isEmpty()) {
                if (isLoggingDebug()) {
                    logDebug("Nothing to do here, all items in the current batch seem to have failed.");
                }

                return 0;
            }

            final JSONObject obj = new JSONObject();
            obj.put(getEndpoint().getLowerCaseName(), jsonObjects);
            obj.put("feedTimestamp", feedTimestamp);
            final StreamRepresentation representation = new StreamRepresentation(MediaType.APPLICATION_JSON) {
                @Override
                public InputStream getStream() throws IOException {
                    throw new UnsupportedOperationException();
                }

                @Override
                public void write(OutputStream outputStream) throws IOException {
                    try {
                        OutputStreamWriter writer = new OutputStreamWriter(outputStream);
                        obj.write(writer);
                        writer.flush();
                    } catch (JSONException ex) {
                        throw new IOException("Cannot write JSON", ex);
                    }
                }
            };
            final Request request = new Request(Method.PUT, endpointUrl,
                    new EncodeRepresentation(Encoding.GZIP, representation));
            Response response = null;

            try {
                response = getProductService().handle(request);
                if (!response.getStatus().equals(Status.SUCCESS_CREATED)) {
                    if (isLoggingInfo()) {
                        logInfo("Sending " + itemDescriptorName + "[" + getIdsFromItemsArray(itemList)
                                + "] failed with status: " + response.getStatus() + " ["
                                + errorResponseToString(response.getEntity()) + "]");
                    }

                    return 0;
                }
            } finally {
                if (response != null) {
                    response.release();
                }
                if (request != null) {
                    request.release();
                }
            }

            return sent;
        } catch (JSONException ex) {
            if (isLoggingInfo()) {
                logInfo("Cannot create JSON representation for " + itemDescriptorName + " info ["
                        + getIdsFromItemsArray(itemList) + "]");
            }

            return 0;
        }
    }

    /**
     * Sends a commit request to the API.
     * @throws IOException if the commit fails.
     */
    protected void sendCommit() throws IOException {
        String commitEndpointUrl = productService.getUrl4Endpoint(getEndpoint(), "commit");

        final Request request = new Request(Method.POST, commitEndpointUrl);
        Response response = null;

        try {
            response = getProductService().handle(request);
            if (!response.getStatus().equals(Status.SUCCESS_OK)) {
                throw new IOException("Failed to send commit with status " + response.getStatus() + " "
                        + errorResponseToString(response.getEntity()));
            }
        } finally {
            if (response != null) {
                response.release();
            }
            if (request != null) {
                request.release();
            }
        }
    }

    /**
     * Sends a rollback request to the API.
     * @throws IOException if the rollback fails.
     */
    protected void sendRollback() throws IOException {
        String rollbackEndpointUrl = productService.getUrl4Endpoint(getEndpoint(), "rollback");

        final Request request = new Request(Method.POST, rollbackEndpointUrl);
        Response response = null;

        try {
            response = getProductService().handle(request);
            if (!response.getStatus().equals(Status.SUCCESS_OK)) {
                throw new IOException("Failed to send rollback with status " + response.getStatus() + " "
                        + errorResponseToString(response.getEntity()));
            }
        } finally {
            if (response != null) {
                response.release();
            }
            if (request != null) {
                request.release();
            }
        }
    }

    /**
     * Sends a delete by query request to the API.
     * @throws IOException if the delete fails.
     */
    private void sendDeleteByQuery() throws IOException {
        String deleteEndpointUrl = endpointUrl;
        deleteEndpointUrl += (getProductService().getPreview()) ? "&" : "?";
        deleteEndpointUrl += "query=*:*";

        final Request request = new Request(Method.DELETE, deleteEndpointUrl);
        Response response = null;

        try {
            response = getProductService().handle(request);
            if (!response.getStatus().equals(Status.SUCCESS_OK)) {
                throw new IOException("Failed to send delete by query with status " + response.getStatus() + " "
                        + errorResponseToString(response.getEntity()));
            }
        } finally {
            if (response != null) {
                response.release();
            }
            if (request != null) {
                request.release();
            }
        }
    }

    public void sendDelete(long feedTimestamp) {
        String deleteEndpointUrl = endpointUrl;
        deleteEndpointUrl += (getProductService().getPreview()) ? "&" : "?";
        deleteEndpointUrl += "feedTimestamp=" + feedTimestamp;

        final Request request = new Request(Method.DELETE, deleteEndpointUrl);
        Response response = null;

        try {
            response = getProductService().handle(request);
            if (isLoggingInfo()) {
                if (response.getStatus().equals(Status.SUCCESS_NO_CONTENT)) {
                    logInfo("Successfully deleted " + itemDescriptorName + " items with feed timestamp before to "
                            + feedTimestamp);
                } else {
                    logInfo("Deleting " + itemDescriptorName + " items with feed timestamp before to "
                            + feedTimestamp + " failed with status: " + response.getStatus());
                }
            }
        } finally {
            if (response != null) {
                response.release();
            }
            if (request != null) {
                request.release();
            }
        }
    }

    /**
     * Get the IDs of all given repository items.
     * @param items Items to get the IDs from.
     * @return List of all IDs concatenated and separated by comma.
     */
    private String getIdsFromItemsArray(RepositoryItem[] items) {
        if (items == null || items.length == 0) {
            return StringUtils.EMPTY;
        }

        StringBuilder buffer = new StringBuilder();

        for (RepositoryItem item : items) {
            buffer.append(item.getRepositoryId()).append(", ");
        }

        //Remove extra comma
        buffer.setLength(buffer.length() - 2);
        return buffer.toString();
    }

    /**
     * Create a printable string out of an HTTP (restlet) error response.
     * @param representation The restlet representation of the error response.
     * @return A printable string containing the HTTP (restlet) error response.
     */
    private String errorResponseToString(Representation representation) {
        String message = "unknown exception";
        try {
            String text = null;
            if (representation != null && (text = representation.getText()) != null) {
                JSONObject obj = new JSONObject(text);
                message = obj.getString("message");

                if (isLoggingDebug() && obj.has("detail")) {
                    message += "\n\n" + obj.getString("detail");
                }
            }
        } catch (JSONException ex) {
            if (isLoggingError()) {
                logError("Can't parse error response.", ex);
            }
        } catch (IOException ex) {
            if (isLoggingError()) {
                logError("Can't get error response.", ex);
            }
        }

        return message;
    }

    /**
     * The mappings for custom properties. For example, category.name -> alias
     * @return custom property mappings
     */
    public Map<String, String> getCustomPropertyMappings() {
        return customPropertyMappings;
    }

    /**
     * Sets the list of custom property mappings
     * @param customPropertyMappings
     */
    public void setCustomPropertyMappings(Map<String, String> customPropertyMappings) {
        this.customPropertyMappings = customPropertyMappings;
    }

    protected Map<String, Map<String, String>> getItemDescriptorCustomPropertyMappings() {
        return itemDescriptorCustomPropertyMappings;
    }

    /**
     * Return the Endpoint for this feed
     * @return an Endpoint enum representing the endpoint for this feed
     */
    public abstract Endpoint getEndpoint();

    /**
     * Convert the given repository item to its corresponding JSON API format.
     * @param item Repository item to convert.
     * @return The JSON representation of the given repository item, or null if there are missing fields.
     * @throws JSONException if there are format issues when creating the JSON object.
     * @throws RepositoryException if item data from the list can't be read.
     */
    protected abstract JSONObject repositoryItemToJson(RepositoryItem item)
            throws JSONException, RepositoryException;

    /**
     * Return a list of required fields when transforming a repository item to JSON.
     * <p/>
     * This list is used for logging purposes only.
     * @return List of required fields when transforming a repository item to JSON, required for logging purposes.
     */
    protected abstract String[] getRequiredItemFields();
}