org.commonjava.maven.ext.io.rest.DefaultTranslator.java Source code

Java tutorial

Introduction

Here is the source code for org.commonjava.maven.ext.io.rest.DefaultTranslator.java

Source

/*
 * Copyright (C) 2012 Red Hat, Inc. (jcasey@redhat.com)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.commonjava.maven.ext.io.rest;

import com.mashape.unirest.http.HttpResponse;
import com.mashape.unirest.http.ObjectMapper;
import com.mashape.unirest.http.Unirest;
import com.mashape.unirest.http.exceptions.UnirestException;
import org.apache.commons.codec.binary.Base32;
import org.commonjava.maven.atlas.ident.ref.ProjectRef;
import org.commonjava.maven.atlas.ident.ref.ProjectVersionRef;
import org.commonjava.maven.ext.common.util.ListUtils;
import org.commonjava.maven.ext.io.rest.exception.RestException;
import org.commonjava.maven.ext.io.rest.mapper.ListingBlacklistMapper;
import org.commonjava.maven.ext.io.rest.mapper.ReportGAVMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;

import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.Random;

import static org.apache.commons.lang.StringUtils.isNotBlank;
import static org.apache.commons.lang.StringUtils.isNotEmpty;
import static org.apache.http.HttpStatus.SC_OK;

/**
 * @author ncross@redhat.com
 * @author vdedik@redhat.com
 * @author jsenko@redhat.com
 */
public class DefaultTranslator implements Translator {
    private static final String REPORTS_LOOKUP_GAVS = "reports/lookup/gavs";

    private static final String LISTING_BLACKLIST_GA = "listings/blacklist/ga";

    private static final Random RANDOM = new Random();

    private static final Base32 CODEC = new Base32();

    private final Logger logger = LoggerFactory.getLogger(getClass());

    private final String endpointUrl;

    private final ReportGAVMapper rgm;

    private final int initialRestMaxSize;

    private final int initialRestMinSize;

    private final ListingBlacklistMapper lbm;

    /**
     * @param endpointUrl is the URL to talk to.
     * @param protocol determines what REST format PME should use. The two formats
     *                 currently available are:
     * @param restMaxSize initial (maximum) size of the rest call; if zero will send everything.
     * @param restMinSize minimum size for the call
     * @param repositoryGroup the group to pass to the endpoint.
     * @param incrementalSerialSuffix the suffix to pass to the endpoint.
     */
    public DefaultTranslator(String endpointUrl, RestProtocol protocol, int restMaxSize, int restMinSize,
            String repositoryGroup, String incrementalSerialSuffix) {
        this.rgm = new ReportGAVMapper(protocol, repositoryGroup, incrementalSerialSuffix);
        this.lbm = new ListingBlacklistMapper(protocol);
        this.endpointUrl = endpointUrl + (isNotBlank(endpointUrl) ? endpointUrl.endsWith("/") ? "" : "/" : "");
        this.initialRestMaxSize = restMaxSize;
        this.initialRestMinSize = restMinSize;
    }

    private void init(ObjectMapper objectMapper) {
        // According to https://github.com/Mashape/unirest-java the default connection timeout is 10000
        // and the default socketTimeout is 60000.
        // We have increased the first to 30 seconds and the second to 10 minutes.
        Unirest.setTimeouts(30000, 600000);
        Unirest.setObjectMapper(objectMapper);
    }

    @Override
    public List<ProjectVersionRef> findBlacklisted(ProjectRef ga) {
        init(lbm);

        final String blacklistEndpointUrl = endpointUrl + LISTING_BLACKLIST_GA;
        List<ProjectVersionRef> result;
        HttpResponse<List> r;

        logger.trace("Called findBlacklisted to {} with {}", blacklistEndpointUrl, ga);

        try {
            r = Unirest.get(blacklistEndpointUrl).header("accept", "application/json")
                    .header("Content-Type", "application/json").header("Log-Context", getHeaderContext())
                    .queryString("groupid", ga.getGroupId()).queryString("artifactid", ga.getArtifactId())
                    .asObject(List.class);

            int status = r.getStatus();
            if (status == SC_OK) {
                result = r.getBody();
            } else {
                throw new RestException(String.format("Failed to establish blacklist calling %s with error %s",
                        this.endpointUrl, lbm.getErrorString()));
            }
        } catch (UnirestException e) {
            throw new RestException("Unable to contact DA", e);
        }

        return result;
    }

    /**
     * Translate the versions.
     * <pre>{@code
     * [ {
     *     "groupId": "com.google.guava",
     *     "artifactId": "guava",
     *     "version": "13.0.1"
     * } }
     * }</pre>
     * This equates to a List of ProjectVersionRef.
     *
     * <pre>{@code
     * {
     *     "productNames": [],
     *     "productVersionIds": [],
     *     "repositoryGroup": "",
     *     "gavs": [
     *     {
     *         "groupId": "com.google.guava",
     *         "artifactId": "guava",
     *         "version": "13.0.1"
     *     } ]
     * }
     * }</pre>
     * There may be a lot of them, possibly causing timeouts or other issues.
     * This is mitigated by splitting them into smaller chunks when an error occurs and retrying.
     */
    public Map<ProjectVersionRef, String> translateVersions(List<ProjectVersionRef> projects) {
        init(rgm);

        final Map<ProjectVersionRef, String> result = new HashMap<>();
        final Queue<Task> queue = new ArrayDeque<>();
        if (initialRestMaxSize != 0) {
            // Presplit
            final List<List<ProjectVersionRef>> partition = ListUtils.partition(projects, initialRestMaxSize);
            for (List<ProjectVersionRef> p : partition) {
                queue.add(new Task(rgm, p, endpointUrl + REPORTS_LOOKUP_GAVS));
            }
            logger.debug("For initial sizing of {} have split the queue into {} ", initialRestMaxSize,
                    queue.size());
        } else {
            queue.add(new Task(rgm, projects, endpointUrl + REPORTS_LOOKUP_GAVS));
        }

        while (!queue.isEmpty()) {
            Task task = queue.remove();
            task.executeTranslate();
            if (task.isSuccess()) {
                result.putAll(task.getResult());
            } else {
                if (task.canSplit() && task.getStatus() == 504) {
                    List<Task> tasks = task.split();

                    logger.warn(
                            "Failed to translate versions for task @{} due to {}, splitting and retrying. Chunk size was: {} and new chunk size {} in {} segments.",
                            task.hashCode(), task.getStatus(), task.getChunkSize(), tasks.get(0).getChunkSize(),
                            tasks.size());
                    queue.addAll(tasks);
                } else {
                    if (task.getStatus() < 0) {
                        logger.debug("Caught exception calling server with message {}", task.getErrorMessage());
                    } else {
                        logger.debug("Did not get status {} but received {}", SC_OK, task.getStatus());
                    }

                    if (task.getStatus() > 0) {
                        throw new RestException("Received response status " + task.getStatus() + " with message: "
                                + task.getErrorMessage());
                    } else {
                        throw new RestException("Received response status " + task.getStatus() + " with message "
                                + task.getErrorMessage());
                    }
                }
            }
        }
        return result;
    }

    private String getHeaderContext() {
        String headerContext;

        if (isNotEmpty(MDC.get("LOG-CONTEXT"))) {
            headerContext = MDC.get("LOG-CONTEXT");
        } else {
            // If we have no MDC PME has been used as the entry point. Dummy one up for DA.
            byte[] randomBytes = new byte[20];
            RANDOM.nextBytes(randomBytes);
            headerContext = "pme-" + CODEC.encodeAsString(randomBytes);
        }

        return headerContext;
    }

    private class Task {
        private List<ProjectVersionRef> chunk;

        private Map<ProjectVersionRef, String> result = null;

        private int status = -1;

        private Exception exception;

        private String errorString;

        private String endpointUrl;

        private ReportGAVMapper pvrm;

        Task(ReportGAVMapper pvrm, List<ProjectVersionRef> chunk, String endpointUrl) {
            this.pvrm = pvrm;
            this.chunk = chunk;
            this.endpointUrl = endpointUrl;
        }

        void executeTranslate() {
            HttpResponse<Map> r;

            try {
                r = Unirest.post(this.endpointUrl).header("accept", "application/json")
                        .header("Content-Type", "application/json").header("Log-Context", getHeaderContext())
                        .body(chunk).asObject(Map.class);

                status = r.getStatus();
                if (status == SC_OK) {
                    this.result = r.getBody();
                } else {
                    errorString = pvrm.getErrorString();
                }
            } catch (UnirestException e) {
                exception = e;
                this.status = -1;
            }
        }

        public List<Task> split() {
            List<Task> res = new ArrayList<>(CHUNK_SPLIT_COUNT);
            if (chunk.size() >= CHUNK_SPLIT_COUNT) {
                // To KISS, overflow the remainder into the last chunk
                int chunkSize = chunk.size() / CHUNK_SPLIT_COUNT;
                for (int i = 0; i < (CHUNK_SPLIT_COUNT - 1); i++) {
                    res.add(new Task(pvrm, chunk.subList(i * chunkSize, (i + 1) * chunkSize), endpointUrl));
                }
                // Last chunk may have different size
                res.add(new Task(pvrm, chunk.subList((CHUNK_SPLIT_COUNT - 1) * chunkSize, chunk.size()),
                        endpointUrl));
            } else {
                for (int i = 0; i < (chunk.size() - initialRestMinSize) + 1; i++) {
                    res.add(new Task(pvrm, chunk.subList(i * initialRestMinSize, (i + 1) * initialRestMinSize),
                            endpointUrl));
                }
            }
            return res;
        }

        boolean canSplit() {
            return (chunk.size() / initialRestMinSize) > 0 && chunk.size() != 1;
        }

        int getStatus() {
            return status;
        }

        boolean isSuccess() {
            return status == SC_OK;
        }

        public Map<ProjectVersionRef, String> getResult() {
            return result;
        }

        public String getErrorMessage() {
            return (exception != null ? exception.getMessage() + ' ' : "")
                    + (errorString != null ? errorString : "");
        }

        int getChunkSize() {
            return chunk.size();
        }
    }
}