org.artifactory.maven.index.MavenIndexerServiceImpl.java Source code

Java tutorial

Introduction

Here is the source code for org.artifactory.maven.index.MavenIndexerServiceImpl.java

Source

/*
 * Artifactory is a binaries repository manager.
 * Copyright (C) 2012 JFrog Ltd.
 *
 * Artifactory is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Artifactory is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with Artifactory.  If not, see <http://www.gnu.org/licenses/>.
 */

package org.artifactory.maven.index;

import com.google.common.base.Predicate;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.lucene.store.FSDirectory;
import org.artifactory.api.config.CentralConfigService;
import org.artifactory.api.context.ContextHelper;
import org.artifactory.common.ArtifactoryHome;
import org.artifactory.common.ConstantValues;
import org.artifactory.common.MutableStatusHolder;
import org.artifactory.descriptor.config.CentralConfigDescriptor;
import org.artifactory.descriptor.index.IndexerDescriptor;
import org.artifactory.descriptor.repo.RepoBaseDescriptor;
import org.artifactory.descriptor.repo.RepoDescriptor;
import org.artifactory.descriptor.repo.VirtualRepoDescriptor;
import org.artifactory.io.TempFileStreamHandle;
import org.artifactory.repo.LocalRepo;
import org.artifactory.repo.RealRepo;
import org.artifactory.repo.RemoteRepo;
import org.artifactory.repo.StoringRepo;
import org.artifactory.repo.service.InternalRepositoryService;
import org.artifactory.repo.virtual.VirtualRepo;
import org.artifactory.resource.ResourceStreamHandle;
import org.artifactory.schedule.BaseTaskServiceDescriptorHandler;
import org.artifactory.schedule.Task;
import org.artifactory.schedule.TaskBase;
import org.artifactory.schedule.TaskService;
import org.artifactory.schedule.TaskUtils;
import org.artifactory.spring.InternalContextHelper;
import org.artifactory.spring.Reloadable;
import org.artifactory.util.ExceptionUtils;
import org.artifactory.util.Files;
import org.artifactory.util.Pair;
import org.artifactory.version.CompoundVersionDetails;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.File;
import java.net.SocketTimeoutException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

/**
 * @author Yoav Landman
 */
@Service
@Reloadable(beanClass = InternalMavenIndexerService.class, initAfter = { TaskService.class,
        InternalRepositoryService.class })
public class MavenIndexerServiceImpl implements InternalMavenIndexerService {
    private static final Logger log = LoggerFactory.getLogger(MavenIndexerServiceImpl.class);

    @Autowired
    private TaskService taskService;

    @Autowired
    private CentralConfigService centralConfig;

    @Autowired
    private InternalRepositoryService repositoryService;

    @Override
    public void init() {
        new IndexerSchedulerHandler(getDescriptor(), null).reschedule();
    }

    @Override
    public void reload(CentralConfigDescriptor oldDescriptor) {
        new IndexerSchedulerHandler(getDescriptor(), oldDescriptor.getIndexer()).reschedule();
    }

    private IndexerDescriptor getDescriptor() {
        return centralConfig.getDescriptor().getIndexer();
    }

    @Override
    public void scheduleImmediateIndexing(MutableStatusHolder statusHolder) {
        scheduleIndexer(statusHolder, new MavenIndexerRunSettings(true, false, null));
    }

    @Override
    public void runSpecificIndexer(MutableStatusHolder statusHolder, List<String> repoKeys,
            boolean forceRemoteDownload) {
        scheduleIndexer(statusHolder, new MavenIndexerRunSettings(true, forceRemoteDownload, repoKeys));
    }

    private void scheduleIndexer(MutableStatusHolder statusHolder, MavenIndexerRunSettings settings) {
        taskService.checkCanStartManualTask(MavenIndexerJob.class, statusHolder);
        if (!statusHolder.isError()) {
            try {
                StringBuilder logMessageBuilder = new StringBuilder("Activating indexer ");
                List<String> repoKeys = settings.getRepoKeys();
                if ((repoKeys != null) && !repoKeys.isEmpty()) {
                    logMessageBuilder.append("for repo '").append(Arrays.toString(repoKeys.toArray())).append("' ");
                }
                logMessageBuilder.append("manually");
                log.info(logMessageBuilder.toString());
                TaskBase task = TaskUtils.createManualTask(MavenIndexerJob.class, 0L);
                task.addAttribute(MavenIndexerJob.SETTINGS, settings);
                taskService.startTask(task, true, true);
            } catch (Exception e) {
                log.error("Error scheduling the indexer.", e);
            }
        }
    }

    @Override
    public void destroy() {
        new IndexerSchedulerHandler(null, null).unschedule();
    }

    @Override
    public void convert(CompoundVersionDetails source, CompoundVersionDetails target) {
    }

    @Override
    public void index(MavenIndexerRunSettings settings) {
        log.info("Starting Maven indexing");
        IndexerDescriptor descriptor = getDescriptor();
        if (!settings.isForceRemoteDownload() && !descriptor.isEnabled() && !settings.isManualRun()) {
            log.debug("Indexer is disabled - doing nothing.");
            return;
        }

        Set<? extends RepoDescriptor> includedRepositories;
        List<String> repoKeys = settings.getRepoKeys();
        if ((repoKeys == null) || repoKeys.isEmpty()) {
            includedRepositories = descriptor.getIncludedRepositories();
            if (includedRepositories == null) {
                // Nothing to index
                log.info("Indexer activated but has no repository declared to index - doing nothing.");
                return;
            }
        } else {
            // only calculate the requested repositories
            includedRepositories = calcSpecificReposForIndexing(repoKeys);
        }

        log.info("Starting non virtual repositories indexing...");
        List<RealRepo> indexedRepos = getNonVirtualRepositoriesToIndex(includedRepositories);
        log.info("Non virtual repositories to index: {}", indexedRepos);
        //Do the indexing work
        for (RealRepo indexedRepo : indexedRepos) {
            //Check if we need to stop/suspend
            if (taskService.pauseOrBreak()) {
                log.info("Stopped indexing on demand");
                return;
            }
            MavenIndexManager mavenIndexManager = new MavenIndexManager(indexedRepo);
            try {
                boolean remoteIndexExists = mavenIndexManager.fetchRemoteIndex(settings.isForceRemoteDownload());
                mavenIndexManager.createLocalIndex(settings.getFireTime(), remoteIndexExists);

                //Check again if we need to stop/suspend
                if (taskService.pauseOrBreak()) {
                    log.info("Stopped indexing on demand");
                    return;
                }
                mavenIndexManager.saveIndexFiles();
            } catch (Exception e) {
                //If we failed to index because of a socket timeout, issue a terse warning instead of a complete stack
                //trace
                Throwable cause = ExceptionUtils.getCauseOfTypes(e, SocketTimeoutException.class);
                if (cause != null) {
                    log.warn("Indexing for repo '" + indexedRepo.getKey() + "' failed: " + e.getMessage() + ".");
                } else {
                    //Just report - don't stop indexing of other repos
                    log.error("Indexing for repo '" + indexedRepo.getKey() + "' failed.", e);
                }
            }
        }
        mergeVirtualRepoIndexes(includedRepositories, indexedRepos);
        log.info("Finished Maven indexing...");
    }

    private Set<? extends RepoDescriptor> calcSpecificReposForIndexing(List<String> repoKeys) {
        Set<RepoBaseDescriptor> repos = Sets.newHashSet();
        repos.addAll(repositoryService.getLocalRepoDescriptors());
        repos.addAll(repositoryService.getRemoteRepoDescriptors());
        repos.addAll(getAllVirtualReposExceptGlobal());
        repos.removeIf(descriptor -> !repoKeys.contains(descriptor.getKey()));
        return repos;
    }

    private List<RealRepo> getNonVirtualRepositoriesToIndex(
            @Nullable Set<? extends RepoDescriptor> includedRepositories) {
        List<RealRepo> indexedRepos = repositoryService.getLocalAndRemoteRepositories();
        if (includedRepositories != null) {
            indexedRepos.removeIf(realRepo -> !includedRepositories.contains(realRepo.getDescriptor()));
        }
        return indexedRepos;
    }

    public void mergeVirtualRepoIndexes(@Nonnull Set<? extends RepoDescriptor> includedRepositories,
            List<RealRepo> indexedRepos) {
        List<VirtualRepo> virtualRepos = getVirtualRepos(includedRepositories);
        log.info("Virtual repositories to index: {}", virtualRepos);
        //Keep a list of extracted index dirs for all the local repo indexes for merging
        Map<StoringRepo, FSDirectory> extractedLocalRepoIndexes = new HashMap<>();
        try {
            //Merge virtual repo indexes
            for (VirtualRepo virtualRepo : virtualRepos) {
                //Check if we need to stop/suspend
                if (taskService.pauseOrBreak()) {
                    log.info("Stopped indexing on demand");
                    return;
                }
                Set<LocalRepo> localRepos = new HashSet<>();
                localRepos.addAll(virtualRepo.getResolvedLocalRepos());
                localRepos.addAll(virtualRepo.getResolvedLocalCachedRepos());
                //Create a temp lucene dir and merge each local into it
                ArtifactoryHome artifactoryHome = ContextHelper.get().getArtifactoryHome();
                File dir = Files.createRandomDir(artifactoryHome.getTempWorkDir(), "artifactory.merged-index");
                RepoIndexer indexer = null;
                try {
                    indexer = new RepoIndexer(virtualRepo);
                    indexer.createContext(dir);

                    log.info("Creating virtual repository index '{}'", virtualRepo);
                    //Take the local index from each relevant indexed repo (for remote take local cache)
                    for (RealRepo indexedRepo : indexedRepos) {
                        //Check if we need to stop/suspend
                        if (taskService.pauseOrBreak()) {
                            log.info("Stopped indexing on demand");
                            return;
                        }
                        LocalRepo localRepo = indexedRepo.isLocal() ? (LocalRepo) indexedRepo
                                : ((RemoteRepo) indexedRepo).getLocalCacheRepo();
                        try {
                            //Extract aside the index from the local repo
                            if (localRepos.contains(localRepo)) {
                                log.debug("Merging index of '{}' to index of virtual repo '{}'", localRepo,
                                        virtualRepo);
                                indexer.mergeInto(localRepo, extractedLocalRepoIndexes);
                            }
                        } catch (Exception e) {
                            log.warn("Could not merge index of local repo '{}' into virtual repo '{}'",
                                    localRepo.getKey(), virtualRepo.getKey());
                        }
                    }
                    //Store the index into the virtual repo
                    //Get the last gz and props and store them - we need to return them or create them from the dir
                    Pair<TempFileStreamHandle, TempFileStreamHandle> tempFileStreamHandlesPair = indexer
                            .createIndex(dir, false);
                    ResourceStreamHandle indexHandle = tempFileStreamHandlesPair.getFirst();
                    ResourceStreamHandle properties = tempFileStreamHandlesPair.getSecond();
                    MavenIndexManager mavenIndexManager = new MavenIndexManager(indexer.getRepo(), indexHandle,
                            properties);
                    mavenIndexManager.saveIndexFiles();
                } finally {
                    if (indexer != null) {
                        indexer.removeTempIndexFiles(dir);
                    }
                    org.apache.commons.io.FileUtils.deleteQuietly(dir);
                }
            }
        } catch (Exception e) {
            log.error("Could not merge virtual repository indexes.", e);
        } finally {
            //Delete temp extracted dirs
            for (FSDirectory directory : extractedLocalRepoIndexes.values()) {
                org.apache.commons.io.FileUtils.deleteQuietly(directory.getFile());
            }
        }
    }

    /**
     * Returns a filtered list of virtual repositories based on the excluded repository list
     *
     * @param includedRepositories List of repositories to index
     * @return List of virtual repositories to index
     */
    private List<VirtualRepo> getVirtualRepos(@Nonnull Set<? extends RepoDescriptor> includedRepositories) {
        List<VirtualRepo> virtualRepositories = repositoryService.getVirtualRepositories();
        List<VirtualRepo> virtualRepositoriesCopy = new ArrayList<>(virtualRepositories);
        virtualRepositoriesCopy.removeIf(virtualRepo -> {
            boolean isFilterGlobalRepo = VirtualRepoDescriptor.GLOBAL_VIRTUAL_REPO_KEY.equals(virtualRepo.getKey())
                    && ConstantValues.disableGlobalRepoAccess.getBoolean();

            boolean isIncluded = includedRepositories.stream().map(RepoDescriptor::getKey)
                    .collect(Collectors.toList()).contains(virtualRepo.getKey());

            return isFilterGlobalRepo || !isIncluded;
        });

        return virtualRepositoriesCopy;
    }

    /**
     * Returns the complete list of virtual repository descriptors, apart from the global one (repo)
     *
     * @return List of all virtual repository descriptors apart from the global one
     */
    private List<VirtualRepoDescriptor> getAllVirtualReposExceptGlobal() {
        List<VirtualRepoDescriptor> virtualRepositoriesCopy = new ArrayList<>(
                repositoryService.getVirtualRepoDescriptors());
        VirtualRepoDescriptor dummyGlobal = new VirtualRepoDescriptor();
        dummyGlobal.setKey(VirtualRepoDescriptor.GLOBAL_VIRTUAL_REPO_KEY);
        virtualRepositoriesCopy.remove(dummyGlobal);
        return virtualRepositoriesCopy;
    }

    static class IndexerSchedulerHandler extends BaseTaskServiceDescriptorHandler<IndexerDescriptor> {
        final List<IndexerDescriptor> oldDescriptorHolder = Lists.newArrayList();
        final List<IndexerDescriptor> newDescriptorHolder = Lists.newArrayList();

        IndexerSchedulerHandler(IndexerDescriptor newDesc, IndexerDescriptor oldDesc) {
            if (newDesc != null) {
                newDescriptorHolder.add(newDesc);
            }
            if (oldDesc != null) {
                oldDescriptorHolder.add(oldDesc);
            }
        }

        @Override
        public String jobName() {
            return "Indexer";
        }

        @Override
        public List<IndexerDescriptor> getNewDescriptors() {
            return newDescriptorHolder;
        }

        @Override
        public List<IndexerDescriptor> getOldDescriptors() {
            return oldDescriptorHolder;
        }

        @Override
        public Predicate<Task> getAllPredicate() {
            return new Predicate<Task>() {
                @Override
                public boolean apply(Task input) {
                    return MavenIndexerJob.class.isAssignableFrom(input.getType());
                }
            };
        }

        @Override
        public Predicate<Task> getPredicate(@Nonnull IndexerDescriptor descriptor) {
            return getAllPredicate();
        }

        @Override
        public void activate(@Nonnull IndexerDescriptor descriptor, boolean manual) {
            String cronExp = descriptor.getCronExp();
            if (descriptor.isEnabled() && cronExp != null) {
                TaskBase task = TaskUtils.createCronTask(MavenIndexerJob.class, cronExp);
                // Passing null for repo keys because they are taken from the indexer descriptor
                MavenIndexerRunSettings settings = new MavenIndexerRunSettings(false, false, null);
                task.addAttribute(MavenIndexerJob.SETTINGS, settings);
                InternalContextHelper.get().getBean(TaskService.class).startTask(task, false, manual);
                log.info("Indexer activated with cron expression '{}'.", cronExp);
            } else {
                log.debug("No indexer cron expression is configured. Indexer will be disabled.");
            }
        }

        @Override
        public IndexerDescriptor findOldFromNew(@Nonnull IndexerDescriptor newDescriptor) {
            return oldDescriptorHolder.isEmpty() ? null : oldDescriptorHolder.get(0);
        }
    }
}