org.jboss.pnc.mavenrepositorymanager.MavenRepositorySession.java Source code

Java tutorial

Introduction

Here is the source code for org.jboss.pnc.mavenrepositorymanager.MavenRepositorySession.java

Source

/**
 * JBoss, Home of Professional Open Source.
 * Copyright 2014 Red Hat, Inc., and individual contributors
 * as indicated by the @author tags.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.jboss.pnc.mavenrepositorymanager;

import org.apache.commons.lang.StringUtils;
import org.commonjava.indy.client.core.Indy;
import org.commonjava.indy.client.core.IndyClientException;
import org.commonjava.indy.client.core.module.IndyContentClientModule;
import org.commonjava.indy.folo.client.IndyFoloAdminClientModule;
import org.commonjava.indy.folo.dto.TrackedContentDTO;
import org.commonjava.indy.folo.dto.TrackedContentEntryDTO;
import org.commonjava.indy.model.core.AccessChannel;
import org.commonjava.indy.model.core.StoreKey;
import org.commonjava.indy.model.core.StoreType;
import org.commonjava.indy.promote.client.IndyPromoteClientModule;
import org.commonjava.indy.promote.model.GroupPromoteRequest;
import org.commonjava.indy.promote.model.GroupPromoteResult;
import org.commonjava.indy.promote.model.PathsPromoteRequest;
import org.commonjava.indy.promote.model.PathsPromoteResult;
import org.commonjava.indy.promote.model.ValidationResult;
import org.commonjava.maven.atlas.ident.ref.ArtifactRef;
import org.commonjava.maven.atlas.ident.ref.SimpleArtifactRef;
import org.commonjava.maven.atlas.ident.util.ArtifactPathInfo;
import org.jboss.pnc.model.Artifact;
import org.jboss.pnc.model.ArtifactRepo;
import org.jboss.pnc.spi.repositorymanager.RepositoryManagerException;
import org.jboss.pnc.spi.repositorymanager.RepositoryManagerResult;
import org.jboss.pnc.spi.repositorymanager.RepositoryManagerStatus;
import org.jboss.pnc.spi.repositorymanager.model.RepositoryConnectionInfo;
import org.jboss.pnc.spi.repositorymanager.model.RepositorySession;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.validation.ConstraintViolation;
import javax.validation.Validation;
import javax.validation.Validator;
import java.io.File;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * {@link RepositorySession} implementation that works with the Maven {@link RepositoryManagerDriver} (which connects to an
 * Indy server instance for repository management). This session contains connection information for rendering Maven
 * settings.xml files and the like, along with the components necessary to extract the artifacts (dependencies, build uploads)
 * for the associated build.
 *
 * Artifact extraction also implies promotion of imported dependencies to a shared-imports Maven repository for safe keeping. In
 * the case of composed (chained) builds, it also implies promotion of the build output to the associated build-set repository
 * group, to expose them for use in successive builds in the chain.
 */
public class MavenRepositorySession implements RepositorySession {

    private Set<String> ignoredPathSuffixes;

    private Indy indy;
    private final String buildContentId;
    private List<String> internalRepoPatterns;

    private final RepositoryConnectionInfo connectionInfo;
    private boolean isSetBuild;

    private Validator validator = Validation.buildDefaultValidatorFactory().getValidator();

    private String buildPromotionGroup;

    // TODO: Create and pass in suitable parameters to Indy to create the
    //       proxy repository.
    @Deprecated
    public MavenRepositorySession(Indy indy, String buildContentId, boolean isSetBuild,
            MavenRepositoryConnectionInfo info) {
        this.indy = indy;
        this.buildContentId = buildContentId;
        this.isSetBuild = isSetBuild;
        this.connectionInfo = info;
    }

    public MavenRepositorySession(Indy indy, String buildContentId, MavenRepositoryConnectionInfo info,
            List<String> internalRepoPatterns, Set<String> ignoredPathSuffixes, String buildPromotionGroup) {
        this.indy = indy;
        this.buildContentId = buildContentId;
        this.internalRepoPatterns = internalRepoPatterns;
        this.ignoredPathSuffixes = ignoredPathSuffixes;
        this.isSetBuild = false; //TODO remove
        this.connectionInfo = info;
        this.buildPromotionGroup = buildPromotionGroup;
    }

    @Override
    public String toString() {
        return "MavenRepositoryConfiguration " + this.hashCode();
    }

    @Override
    public ArtifactRepo.Type getType() {
        return ArtifactRepo.Type.MAVEN;
    }

    @Override
    public String getBuildRepositoryId() {
        return buildContentId;
    }

    @Override
    public RepositoryConnectionInfo getConnectionInfo() {
        return connectionInfo;
    }

    /**
     * Retrieve tracking report from repository manager. Add each tracked download to the dependencies of the build result. Add
     * each tracked upload to the built artifacts of the build result. Promote uploaded artifacts to the product-level storage.
     * Finally, clear the tracking report, and delete the hosted repository + group associated with the completed build.
     */
    @Override
    public RepositoryManagerResult extractBuildArtifacts() throws RepositoryManagerException {
        TrackedContentDTO report;
        try {
            IndyFoloAdminClientModule foloAdmin = indy.module(IndyFoloAdminClientModule.class);
            boolean sealed = foloAdmin.sealTrackingRecord(buildContentId);
            if (!sealed) {
                throw new RepositoryManagerException("Failed to seal content-tracking record for: %s.",
                        buildContentId);
            }

            report = foloAdmin.getTrackingReport(buildContentId);
        } catch (IndyClientException e) {
            throw new RepositoryManagerException("Failed to retrieve tracking report for: %s. Reason: %s", e,
                    buildContentId, e.getMessage());
        }
        if (report == null) {
            throw new RepositoryManagerException("Failed to retrieve tracking report for: %s.", buildContentId);
        }

        Comparator<Artifact> comp = (one, two) -> one.getIdentifier().compareTo(two.getIdentifier());

        List<Artifact> uploads = processUploads(report);
        Collections.sort(uploads, comp);

        List<Artifact> downloads = processDownloads(report);
        Collections.sort(downloads, comp);

        try {
            indy.stores().delete(StoreType.group, buildContentId,
                    "[Post-Build] Removing build aggregation group: " + buildContentId);
        } catch (IndyClientException e) {
            throw new RepositoryManagerException("Failed to retrieve AProx stores module. Reason: %s", e,
                    e.getMessage());
        }

        Logger logger = LoggerFactory.getLogger(getClass());
        logger.info("Returning built artifacts / dependencies:\nUploads:\n  {}\n\nDownloads:\n  {}\n\n",
                StringUtils.join(uploads, "\n  "), StringUtils.join(downloads, "\n  "));

        String log = "";
        RepositoryManagerStatus status = RepositoryManagerStatus.SUCCESS;
        try {
            promoteToBuildContentSet();
        } catch (RepositoryManagerException rme) {
            status = RepositoryManagerStatus.VALIDATION_ERROR;
            log = rme.getMessage();
            logger.error("Promotion validation error(s): \n" + log);
        }

        return new MavenRepositoryManagerResult(uploads, downloads, buildContentId, log, status);
    }

    /**
     * Promote all build dependencies NOT ALREADY CAPTURED to the hosted repository holding store for the shared imports and
     * return dependency artifacts meta data.
     *
     * @param report The tracking report that contains info about artifacts downloaded by the build
     * @return List of dependency artifacts meta data
     * @throws RepositoryManagerException In case of a client API transport error or an error during promotion of artifacts
     */
    private List<Artifact> processDownloads(TrackedContentDTO report) throws RepositoryManagerException {
        Logger logger = LoggerFactory.getLogger(getClass());

        IndyContentClientModule content;
        try {
            content = indy.content();
        } catch (IndyClientException e) {
            throw new RepositoryManagerException("Failed to retrieve AProx client module. Reason: %s", e,
                    e.getMessage());
        }

        List<Artifact> deps = new ArrayList<>();

        Set<TrackedContentEntryDTO> downloads = report.getDownloads();
        if (downloads != null) {

            Map<StoreKey, Set<String>> toPromote = new HashMap<>();

            StoreKey sharedImports = new StoreKey(StoreType.hosted, MavenRepositoryConstants.SHARED_IMPORTS_ID);
            //            StoreKey sharedReleases = new StoreKey(StoreType.hosted, RepositoryManagerDriver.SHARED_RELEASES_ID);

            for (TrackedContentEntryDTO download : downloads) {
                String path = download.getPath();
                if (ignoreContent(path)) {
                    logger.debug("Ignoring download (matched in ignored-suffixes): {} (From: {})",
                            download.getPath(), download.getStoreKey());
                    continue;
                }

                StoreKey sk = download.getStoreKey();

                // If the entry is from a hosted repository, it shouldn't be auto-promoted.
                // If the entry is already in shared-imports, it shouldn't be auto-promoted to there.
                // New binary imports will be coming from a remote repository...
                // TODO: Enterprise maven repository (product repo) handling...
                if (isExternalOrigin(sk) && StoreType.hosted != sk.getType()) {
                    switch (download.getAccessChannel()) {
                    case MAVEN_REPO:
                        // this has not been captured, so promote it.
                        Set<String> paths = toPromote.get(sk);
                        if (paths == null) {
                            paths = new HashSet<>();
                            toPromote.put(sk, paths);
                        }

                        paths.add(download.getPath());
                        break;

                    default:
                        // do not promote anything else anywhere
                        break;
                    }
                }

                ArtifactPathInfo pathInfo = ArtifactPathInfo.parse(path);

                String identifier;
                if (pathInfo == null) {
                    identifier = download.getOriginUrl();
                    if (identifier == null) {
                        // this is from a hosted repository, either shared-imports or a build, or something like that.
                        identifier = download.getLocalUrl();
                    }
                    identifier += '|' + download.getSha256();
                } else {
                    ArtifactRef aref = new SimpleArtifactRef(pathInfo.getProjectId(), pathInfo.getType(),
                            pathInfo.getClassifier());
                    identifier = aref.toString();
                }

                logger.info("Recording download: {}", identifier);

                String originUrl = download.getOriginUrl();
                if (originUrl == null) {
                    // this is from a hosted repository, either shared-imports or a build, or something like that.
                    originUrl = download.getLocalUrl();
                }

                Artifact.Builder artifactBuilder = Artifact.Builder.newBuilder().md5(download.getMd5())
                        .sha1(download.getSha1()).sha256(download.getSha256()).size(download.getSize())
                        .deployPath(download.getPath()).originUrl(originUrl).importDate(Date.from(Instant.now()))
                        .filename(new File(path).getName()).identifier(identifier)
                        .repoType(toRepoType(download.getAccessChannel()));

                Artifact artifact = validateArtifact(artifactBuilder.build());
                deps.add(artifact);
            }

            for (Map.Entry<StoreKey, Set<String>> entry : toPromote.entrySet()) {
                PathsPromoteRequest req = new PathsPromoteRequest(entry.getKey(), sharedImports, entry.getValue())
                        .setPurgeSource(false);
                doPromoteByPath(req);
            }
        }

        return deps;
    }

    private boolean isExternalOrigin(StoreKey sk) {
        String repoName = sk.getName();
        for (String pattern : internalRepoPatterns) {
            //            Logger logger = LoggerFactory.getLogger(getClass());
            //            logger.info( "Checking ")
            if (pattern.equals(repoName)) {
                return false;
            }

            if (repoName.matches(pattern)) {
                return false;
            }
        }

        return true;
    }

    /**
     * Return output artifacts metadata.
     *
     * @param report The tracking report that contains info about artifacts uploaded (output) from the build
     * @return List of output artifacts meta data
     * @throws RepositoryManagerException In case of a client API transport error or an error during promotion of artifacts
     */
    private List<Artifact> processUploads(TrackedContentDTO report) throws RepositoryManagerException {
        Logger logger = LoggerFactory.getLogger(getClass());

        Set<TrackedContentEntryDTO> uploads = report.getUploads();
        if (uploads != null) {
            List<Artifact> builds = new ArrayList<>();

            for (TrackedContentEntryDTO upload : uploads) {
                String path = upload.getPath();
                if (ignoreContent(path)) {
                    logger.debug("Ignoring upload (matched in ignored-suffixes): {} (From: {})", path,
                            upload.getStoreKey());
                    continue;
                }

                ArtifactPathInfo pathInfo = ArtifactPathInfo.parse(path);

                String identifier;
                if (pathInfo == null) {
                    identifier = upload.getOriginUrl();
                    if (identifier == null) {
                        // this is to a hosted repository, either the build repo or something like that.
                        identifier = upload.getLocalUrl();
                    }
                    identifier += '|' + upload.getSha256();
                } else {
                    ArtifactRef aref = new SimpleArtifactRef(pathInfo.getProjectId(), pathInfo.getType(),
                            pathInfo.getClassifier());
                    identifier = aref.toString();
                }

                logger.info("Recording upload: {}", identifier);

                Artifact.Builder artifactBuilder = Artifact.Builder.newBuilder().md5(upload.getMd5())
                        .sha1(upload.getSha1()).sha256(upload.getSha256()).size(upload.getSize())
                        .deployPath(upload.getPath()).filename(new File(path).getName()).identifier(identifier)
                        .repoType(ArtifactRepo.Type.MAVEN);

                Artifact artifact = validateArtifact(artifactBuilder.build());
                builds.add(artifact);
            }

            return builds;
        }
        return Collections.emptyList();
    }

    /**
     * Check artifact for any validation errors.  If there are constraint violations, then a RepositoryManagerException is thrown.
     * Otherwise the artifact is returned.
     *
     * @param artifact to validate
     * @return the same artifact
     * @throws RepositoryManagerException if there are constraint violations
     */
    private Artifact validateArtifact(Artifact artifact) throws RepositoryManagerException {
        Set<ConstraintViolation<Artifact>> violations = validator.validate(artifact);
        if (!violations.isEmpty()) {
            throw new RepositoryManagerException("Repository manager returned invalid artifact: "
                    + artifact.toString() + " Constraint Violations: %s", violations);
        }
        return artifact;
    }

    /**
     * Promotes a set of artifact paths (or everything, if the path-set is missing) from a particular AProx artifact store to
     * another, and handle the various error conditions that may arise. If the promote call fails, attempt to rollback before
     * throwing an exception.
     *
     * @param req The promotion request to process, which contains source and target store keys, and (optionally) the set of
     *        paths to promote
     * @throws RepositoryManagerException When either the client API throws an exception due to something unexpected in
     *         transport, or if the promotion process results in an error.
     */
    private void doPromoteByPath(PathsPromoteRequest req) throws RepositoryManagerException {
        IndyPromoteClientModule promoter;
        try {
            promoter = indy.module(IndyPromoteClientModule.class);
        } catch (IndyClientException e) {
            throw new RepositoryManagerException("Failed to retrieve AProx client module. Reason: %s", e,
                    e.getMessage());
        }

        try {
            PathsPromoteResult result = promoter.promoteByPath(req);
            if (result.getError() != null) {
                String addendum = "";
                try {
                    PathsPromoteResult rollback = promoter.rollbackPathPromote(result);
                    if (rollback.getError() != null) {
                        addendum = "\nROLLBACK WARNING: Promotion rollback also failed! Reason given: "
                                + result.getError();
                    }

                } catch (IndyClientException e) {
                    throw new RepositoryManagerException("Rollback failed for promotion of: %s. Reason: %s", e, req,
                            e.getMessage());
                }

                throw new RepositoryManagerException("Failed to promote: %s. Reason given was: %s%s", req,
                        result.getError(), addendum);
            }
        } catch (IndyClientException e) {
            throw new RepositoryManagerException("Failed to promote: %s. Reason: %s", e, req, e.getMessage());
        }
    }

    /**
     * Promote the build output to shared-releases (using group promotion, where the build repo is added to the group's
     * membership).
     */
    public void promoteToBuildContentSet() throws RepositoryManagerException {
        IndyPromoteClientModule promoter;
        try {
            promoter = indy.module(IndyPromoteClientModule.class);
        } catch (IndyClientException e) {
            throw new RepositoryManagerException("Failed to retrieve AProx client module. Reason: %s", e,
                    e.getMessage());
        }

        GroupPromoteRequest request = new GroupPromoteRequest(new StoreKey(StoreType.hosted, buildContentId),
                buildPromotionGroup);
        try {
            GroupPromoteResult result = promoter.promoteToGroup(request);
            if (!result.succeeded()) {
                String reason = result.getError();
                if (reason == null) {
                    ValidationResult validations = result.getValidations();
                    if (validations != null) {
                        StringBuilder sb = new StringBuilder();
                        sb.append("One or more validation rules failed in rule-set ")
                                .append(validations.getRuleSet()).append(":\n");

                        validations.getValidatorErrors().forEach((rule, error) -> {
                            sb.append("- ").append(rule).append(":\n").append(error).append("\n\n");
                        });

                        reason = sb.toString();
                    }
                }

                throw new RepositoryManagerException("Failed to promote: %s to group: %s. Reason given was: %s",
                        request.getSource(), request.getTargetGroup(), reason);
            }
        } catch (IndyClientException e) {
            throw new RepositoryManagerException("Failed to promote: %s. Reason: %s", e, request, e.getMessage());
        }
    }

    private boolean ignoreContent(String path) {
        for (String suffix : ignoredPathSuffixes) {
            if (path.endsWith(suffix))
                return true;
        }

        return false;
    }

    private ArtifactRepo.Type toRepoType(AccessChannel accessChannel) {
        switch (accessChannel) {
        case MAVEN_REPO:
            return ArtifactRepo.Type.MAVEN;
        case GENERIC_PROXY:
            return ArtifactRepo.Type.GENERIC_PROXY;
        default:
            return ArtifactRepo.Type.GENERIC_PROXY;
        }
    }
}