Java tutorial
/** * JBoss, Home of Professional Open Source. * Copyright 2014-2019 Red Hat, Inc., and individual contributors * as indicated by the @author tags. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.jboss.pnc.indyrepositorymanager; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.commonjava.indy.client.core.Indy; import org.commonjava.indy.client.core.IndyClientException; import org.commonjava.indy.client.core.IndyClientHttp; import org.commonjava.indy.client.core.IndyClientModule; import org.commonjava.indy.client.core.auth.IndyClientAuthenticator; import org.commonjava.indy.client.core.auth.OAuth20BearerTokenAuthenticator; import org.commonjava.indy.folo.client.IndyFoloAdminClientModule; import org.commonjava.indy.folo.client.IndyFoloContentClientModule; import org.commonjava.indy.model.core.Group; import org.commonjava.indy.model.core.HostedRepository; import org.commonjava.indy.model.core.RemoteRepository; import org.commonjava.indy.model.core.StoreKey; import org.commonjava.indy.model.core.StoreType; import org.commonjava.indy.model.core.dto.StoreListingDTO; import org.commonjava.indy.model.core.io.IndyObjectMapper; import org.commonjava.indy.promote.client.IndyPromoteClientModule; import org.commonjava.util.jhttpc.model.SiteConfig; import org.commonjava.util.jhttpc.model.SiteConfigBuilder; import org.jboss.pnc.common.Configuration; import org.jboss.pnc.common.json.ConfigurationParseException; import org.jboss.pnc.common.json.moduleconfig.IndyRepoDriverModuleConfig; import org.jboss.pnc.common.json.moduleconfig.IndyRepoDriverModuleConfig.IgnoredPathSuffixes; import org.jboss.pnc.common.json.moduleconfig.IndyRepoDriverModuleConfig.InternalRepoPatterns; import org.jboss.pnc.common.json.moduleprovider.PncConfigProvider; import org.jboss.pnc.model.BuildRecord; import org.jboss.pnc.spi.repositorymanager.ArtifactRepository; import org.jboss.pnc.spi.repositorymanager.BuildExecution; import org.jboss.pnc.spi.repositorymanager.RepositoryManager; import org.jboss.pnc.spi.repositorymanager.RepositoryManagerException; import org.jboss.pnc.spi.repositorymanager.model.RepositorySession; import org.jboss.pnc.spi.repositorymanager.model.RunningRepositoryDeletion; import org.jboss.pnc.spi.repositorymanager.model.RunningRepositoryPromotion; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.enterprise.context.ApplicationScoped; import javax.inject.Inject; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import org.jboss.pnc.enums.RepositoryType; import static org.commonjava.indy.pkg.maven.model.MavenPackageTypeDescriptor.MAVEN_PKG_KEY; import static org.commonjava.indy.pkg.npm.model.NPMPackageTypeDescriptor.NPM_PKG_KEY; import static org.jboss.pnc.indyrepositorymanager.IndyRepositoryConstants.COMMON_BUILD_GROUP_CONSTITUENTS_GROUP; import static org.jboss.pnc.indyrepositorymanager.IndyRepositoryConstants.DRIVER_ID; import static org.jboss.pnc.indyrepositorymanager.IndyRepositoryConstants.TEMPORARY_BUILDS_GROUP; /** * Implementation of {@link RepositoryManager} that manages an <a href="https://github.com/jdcasey/indy">Indy</a> instance to * support repositories for Maven-ish builds. * <p> * Created by <a href="mailto:matejonnet@gmail.com">Matej Lazar</a> on 2014-11-25. * * @author <a href="mailto:jdcasey@commonjava.org">John Casey</a> */ @ApplicationScoped public class RepositoryManagerDriver implements RepositoryManager { static final String EXTRA_PUBLIC_REPOSITORIES_KEY = "EXTRA_REPOSITORIES"; private static final Logger userLog = LoggerFactory.getLogger("org.jboss.pnc._userlog_.build-executor"); private final Logger logger = LoggerFactory.getLogger(getClass()); private final int DEFAULT_REQUEST_TIMEOUT; private final String BUILD_PROMOTION_TARGET; private final String TEMP_BUILD_PROMOTION_GROUP; private String baseUrl; private Map<String, Indy> indyMap = new HashMap<>(); private InternalRepoPatterns internalRepoPatterns; private IgnoredPathSuffixes ignoredPathSuffixes; @Deprecated public RepositoryManagerDriver() { // workaround for CDI constructor parameter injection bug this.DEFAULT_REQUEST_TIMEOUT = 0; this.BUILD_PROMOTION_TARGET = ""; this.TEMP_BUILD_PROMOTION_GROUP = ""; } @Inject public RepositoryManagerDriver(Configuration configuration) { IndyRepoDriverModuleConfig config; try { config = configuration.getModuleConfig(new PncConfigProvider<>(IndyRepoDriverModuleConfig.class)); } catch (ConfigurationParseException e) { throw new IllegalStateException("Cannot read configuration for " + DRIVER_ID + ".", e); } this.DEFAULT_REQUEST_TIMEOUT = config.getDefaultRequestTimeout(); this.BUILD_PROMOTION_TARGET = config.getBuildPromotionTarget(); this.TEMP_BUILD_PROMOTION_GROUP = config.getTempBuildPromotionGroup(); baseUrl = StringUtils.stripEnd(config.getBaseUrl(), "/"); if (!baseUrl.endsWith("/api")) { baseUrl += "/api"; } List<String> constInternalRepoPatterns = new ArrayList<>(); constInternalRepoPatterns.add(IndyRepositoryConstants.SHARED_IMPORTS_ID); internalRepoPatterns = new InternalRepoPatterns(); internalRepoPatterns.setMaven(constInternalRepoPatterns); internalRepoPatterns.setNpm(new ArrayList<>(constInternalRepoPatterns)); InternalRepoPatterns extraInternalRepoPatterns = config.getInternalRepoPatterns(); if (extraInternalRepoPatterns != null) { internalRepoPatterns.addMaven(extraInternalRepoPatterns.getMaven()); internalRepoPatterns.addNpm(extraInternalRepoPatterns.getNpm()); } IgnoredPathSuffixes ignoredPathSuffixes = config.getIgnoredPathSuffixes(); if (ignoredPathSuffixes == null) { this.ignoredPathSuffixes = new IgnoredPathSuffixes(); } else { this.ignoredPathSuffixes = ignoredPathSuffixes; // TODO do we need a copy? } } private Indy init(String accessToken) { Indy indy = indyMap.get(accessToken); if (indy == null) { //TODO use indyConcurrentMap.computeIfAbsent IndyClientAuthenticator authenticator = null; if (accessToken != null) { authenticator = new OAuth20BearerTokenAuthenticator(accessToken); } try { SiteConfig siteConfig = new SiteConfigBuilder("indy", baseUrl) .withRequestTimeoutSeconds(DEFAULT_REQUEST_TIMEOUT) .withMaxConnections(IndyClientHttp.GLOBAL_MAX_CONNECTIONS).build(); IndyClientModule[] modules = new IndyClientModule[] { new IndyFoloAdminClientModule(), new IndyFoloContentClientModule(), new IndyPromoteClientModule() }; indy = new Indy(siteConfig, authenticator, new IndyObjectMapper(true), modules); indyMap.put(accessToken, indy); } catch (IndyClientException e) { throw new IllegalStateException("Failed to create Indy client: " + e.getMessage(), e); } } return indy; } @Override public void close(String accessToken) { if (indyMap.containsKey(accessToken)) { IOUtils.closeQuietly(indyMap.get(accessToken)); indyMap.remove(accessToken); } } /** * Supports RepositoryType#MAVEN and RepositoryType#NPM */ @Override public boolean canManage(RepositoryType managerType) { return (managerType == RepositoryType.MAVEN) || (managerType == RepositoryType.NPM); } /** * Use the Indy client API to setup global and build-set level repos and groups, then setup the repo/group needed for this * build. Calculate the URL to use for resolving artifacts using the Indy Folo API (Folo is an artifact activity-tracker). * Return a new session ({@link IndyRepositorySession}) containing this information. * * @throws RepositoryManagerException In the event one or more repositories or groups can't be created to support the build * (or product, or shared-releases). */ @Override public RepositorySession createBuildRepository(BuildExecution buildExecution, String accessToken, String serviceAccountToken, RepositoryType repositoryType, Map<String, String> genericParameters) throws RepositoryManagerException { Indy indy = init(accessToken); Indy serviceAccountIndy = init(serviceAccountToken); String packageType = getIndyPackageTypeKey(repositoryType); String buildId = buildExecution.getBuildContentId(); try { setupBuildRepos(buildExecution, packageType, serviceAccountIndy, genericParameters); } catch (IndyClientException e) { throw new RepositoryManagerException( "Failed to setup repository or repository group for this build: %s", e, e.getMessage()); } // since we're setting up a group/hosted repo per build, we can pin the tracking ID to the build repo ID. String url; String deployUrl; try { // manually initialize the tracking record, just in case (somehow) nothing gets downloaded/uploaded. indy.module(IndyFoloAdminClientModule.class).initReport(buildId); StoreKey groupKey = new StoreKey(packageType, StoreType.group, buildId); url = indy.module(IndyFoloContentClientModule.class).trackingUrl(buildId, groupKey); StoreKey hostedKey = new StoreKey(packageType, StoreType.hosted, buildId); deployUrl = indy.module(IndyFoloContentClientModule.class).trackingUrl(buildId, hostedKey); logger.info("Using '{}' for {} repository access in build: {}", url, packageType, buildId); } catch (IndyClientException e) { throw new RepositoryManagerException( "Failed to retrieve Indy client module for the artifact tracker: %s", e, e.getMessage()); } boolean tempBuild = buildExecution.isTempBuild(); String buildPromotionGroup = tempBuild ? TEMP_BUILD_PROMOTION_GROUP : BUILD_PROMOTION_TARGET; return new IndyRepositorySession(indy, serviceAccountIndy, buildId, packageType, new IndyRepositoryConnectionInfo(url, deployUrl), internalRepoPatterns, ignoredPathSuffixes, buildPromotionGroup, tempBuild); } private String getIndyPackageTypeKey(RepositoryType repoType) { switch (repoType) { case MAVEN: return MAVEN_PKG_KEY; case NPM: return NPM_PKG_KEY; default: throw new IllegalArgumentException( "Repository type " + repoType + " is not supported by this repository manager driver."); } } /** * Create the hosted repository and group necessary to support a single build. The hosted repository holds artifacts * uploaded from the build, and the group coordinates access to this hosted repository, along with content from the * product-level content group with which this build is associated. The group also provides a tracking target, so the * repository manager can keep track of downloads and uploads for the build. * * @param execution The execution object, which contains the content id for creating the repo, and the build id. * @param packageType the package type key used by Indy * @throws IndyClientException */ private void setupBuildRepos(BuildExecution execution, String packageType, Indy indy, Map<String, String> genericParameters) throws IndyClientException { String buildContentId = execution.getBuildContentId(); int id = execution.getId(); // if the build-level group doesn't exist, create it. StoreKey groupKey = new StoreKey(packageType, StoreType.group, buildContentId); if (!indy.stores().exists(groupKey)) { // if the product-level storage repo (for in-progress product builds) doesn't exist, create it. StoreKey hostedKey = new StoreKey(packageType, StoreType.hosted, buildContentId); boolean tempBuild = execution.isTempBuild(); if (!indy.stores().exists(hostedKey)) { HostedRepository buildArtifacts = new HostedRepository(packageType, buildContentId); buildArtifacts.setAllowSnapshots(tempBuild); buildArtifacts.setAllowReleases(true); buildArtifacts.setDescription(String.format("Build output for PNC %s build #%s", packageType, id)); indy.stores().create(buildArtifacts, "Creating hosted repository for " + packageType + " build: " + id + " (repo: " + buildContentId + ")", HostedRepository.class); } Group buildGroup = new Group(packageType, buildContentId); String adjective = tempBuild ? "temporary " : ""; buildGroup.setDescription(String.format("Aggregation group for PNC %sbuild #%s", adjective, id)); // build-local artifacts buildGroup.addConstituent(hostedKey); // Global-level repos, for captured/shared artifacts and access to the outside world addGlobalConstituents(packageType, buildGroup, tempBuild); // add extra repositories removed from poms by the adjust process and set in BC by user List<ArtifactRepository> extraDependencyRepositories = extractExtraRepositoriesFromGenericParameters( genericParameters); if (execution.getArtifactRepositories() != null) { extraDependencyRepositories.addAll(execution.getArtifactRepositories()); } addExtraConstituents(packageType, extraDependencyRepositories, id, buildContentId, indy, buildGroup); indy.stores().create(buildGroup, "Creating repository group for resolving artifacts in build: " + id + " (repo: " + buildContentId + ")", Group.class); } } List<ArtifactRepository> extractExtraRepositoriesFromGenericParameters(Map<String, String> genericParameters) { String extraReposString = genericParameters.get(EXTRA_PUBLIC_REPOSITORIES_KEY); if (extraReposString == null) { return new ArrayList<>(); } return Arrays.stream(extraReposString.split("\n")).map((repoString) -> { try { String id = new URL(repoString).getHost().replaceAll("\\.", "-"); return ArtifactRepository.build(id, id, repoString.trim(), true, false); } catch (MalformedURLException e) { userLog.warn("Malformed repository URL entered: " + repoString + ". Skipping."); return null; } }).filter((x) -> x != null).collect(Collectors.toList()); } /** * Adds extra remote repositories to the build group that are requested for the particular build. For a Maven build * these are repositories defined in the root pom removed by PME by the adjust process. * * @param packageType * the package type key used by Indy * @param repositories * the list of repositories to be added * @param buildId * build ID * @param buildContentId * build content ID * @param indy * Indy client * @param buildGroup * target build group in which the repositories are added * * @throws IndyClientException * in case of an issue when communicating with the repository manager */ private void addExtraConstituents(String packageType, List<ArtifactRepository> repositories, int buildId, String buildContentId, Indy indy, Group buildGroup) throws IndyClientException { if (repositories != null && !repositories.isEmpty()) { StoreListingDTO<RemoteRepository> existingRepos = indy.stores().listRemoteRepositories(packageType); for (ArtifactRepository repository : repositories) { StoreKey remoteKey = null; for (RemoteRepository existingRepo : existingRepos) { if (existingRepo.getUrl().equals(repository.getUrl())) { remoteKey = existingRepo.getKey(); break; } } if (remoteKey == null) { // this is basically an implied repo, so using the same prefix "i-" String remoteName = "i-" + convertIllegalCharacters(repository.getId()); // find a free repository ID for the newly created repo remoteKey = new StoreKey(packageType, StoreType.remote, remoteName); int i = 2; while (indy.stores().exists(remoteKey)) { remoteKey = new StoreKey(packageType, StoreType.remote, remoteName + "-" + i++); } RemoteRepository remoteRepo = new RemoteRepository(packageType, remoteKey.getName(), repository.getUrl()); remoteRepo.setAllowReleases(repository.getReleases()); remoteRepo.setAllowSnapshots(repository.getSnapshots()); remoteRepo.setDescription( "Implicitly created " + packageType + " repo for: " + repository.getName() + " (" + repository.getId() + ") from repository declaration removed by PME in build " + buildId + " (repo: " + buildContentId + ")"); indy.stores().create(remoteRepo, "Creating extra remote repository " + repository.getName() + " (" + repository.getId() + ") for build: " + buildId + " (repo: " + buildContentId + ")", RemoteRepository.class); } buildGroup.addConstituent(remoteKey); } } } /** * Converts characters in a given string considered as illegal by Indy to underscores. * * @param name repository name * @return string with converted characters */ private String convertIllegalCharacters(String name) { char[] result = new char[name.length()]; for (int i = 0; i < name.length(); i++) { char checkedChar = name.charAt(i); if (Character.isLetterOrDigit(checkedChar) || checkedChar == '+' || checkedChar == '-' || checkedChar == '.') { result[i] = checkedChar; } else { result[i] = '_'; } } return String.valueOf(result); } /** * Add the constituents that every build repository group should contain: * <ol> * <li>builds-untested (Group)</li> * <li>for temporary builds add also temporary-builds (Group)</li> * <li>shared-imports (Hosted Repo)</li> * <li>public (Group)</li> * </ol> * @param pakageType package type key used by Indy */ private void addGlobalConstituents(String pakageType, Group group, boolean tempBuild) { // 1. global builds artifacts if (tempBuild) { group.addConstituent(new StoreKey(pakageType, StoreType.group, TEMPORARY_BUILDS_GROUP)); } group.addConstituent(new StoreKey(pakageType, StoreType.group, COMMON_BUILD_GROUP_CONSTITUENTS_GROUP)); } /** * Convenience method for tests. */ protected Indy getIndy(String accessToken) { return init(accessToken); } /** * Promote the hosted repository associated with a given project build to an arbitrary repository group. * * @return The promotion instance, which won't actually trigger promotion until its * {@link RunningRepositoryPromotion#monitor(java.util.function.Consumer, java.util.function.Consumer)} method is * called. */ @Override public RunningRepositoryPromotion promoteBuild(BuildRecord buildRecord, String pakageType, String toGroup, String accessToken) throws RepositoryManagerException { Indy indy = init(accessToken); return new IndyRunningPromotion(pakageType, StoreType.hosted, buildRecord.getBuildContentId(), toGroup, indy); } @Override public RunningRepositoryDeletion deleteBuild(BuildRecord buildRecord, String pakageType, String accessToken) throws RepositoryManagerException { Indy indy = init(accessToken); return new IndyRunningDeletion(pakageType, StoreType.hosted, buildRecord.getBuildContentId(), indy); } }