com.facebook.buck.distributed.DistBuildFileHashes.java Source code

Java tutorial

Introduction

Here is the source code for com.facebook.buck.distributed.DistBuildFileHashes.java

Source

/*
 * Copyright 2016-present Facebook, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License. You may obtain
 * a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations
 * under the License.
 */

package com.facebook.buck.distributed;

import com.facebook.buck.cli.BuckConfig;
import com.facebook.buck.distributed.thrift.BuildJobStateFileHashEntry;
import com.facebook.buck.distributed.thrift.BuildJobStateFileHashes;
import com.facebook.buck.hashing.FileHashLoader;
import com.facebook.buck.io.ArchiveMemberPath;
import com.facebook.buck.io.MorePaths;
import com.facebook.buck.io.ProjectFilesystem;
import com.facebook.buck.rules.ActionGraph;
import com.facebook.buck.rules.BuildRule;
import com.facebook.buck.rules.RuleKey;
import com.facebook.buck.rules.SourcePathResolver;
import com.facebook.buck.rules.SourcePathRuleFinder;
import com.facebook.buck.rules.keys.DefaultRuleKeyFactory;
import com.facebook.buck.util.cache.DefaultFileHashCache;
import com.facebook.buck.util.cache.FileHashCache;
import com.facebook.buck.util.cache.StackedFileHashCache;
import com.google.common.base.Function;
import com.google.common.base.Functions;
import com.google.common.base.Throwables;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;

import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;

/**
 * Responsible for extracting file hash and {@link RuleKey} information from the {@link ActionGraph}
 * and presenting it as a Thrift data structure.
 */
public class DistBuildFileHashes {
    private final LoadingCache<ProjectFilesystem, BuildJobStateFileHashes> remoteFileHashes;
    private final LoadingCache<ProjectFilesystem, FileHashLoader> fileHashLoaders;
    private final LoadingCache<ProjectFilesystem, DefaultRuleKeyFactory> ruleKeyFactories;

    private final ListenableFuture<ImmutableList<BuildJobStateFileHashes>> fileHashes;
    private final ListenableFuture<ImmutableMap<BuildRule, RuleKey>> ruleKeys;

    public DistBuildFileHashes(ActionGraph actionGraph, final SourcePathResolver sourcePathResolver,
            SourcePathRuleFinder ruleFinder, final FileHashCache rootCellFileHashCache,
            final Function<? super Path, Integer> cellIndexer, ListeningExecutorService executorService,
            final int keySeed, final BuckConfig buckConfig) {

        this.remoteFileHashes = CacheBuilder.newBuilder()
                .build(new CacheLoader<ProjectFilesystem, BuildJobStateFileHashes>() {
                    @Override
                    public BuildJobStateFileHashes load(ProjectFilesystem filesystem) throws Exception {
                        BuildJobStateFileHashes fileHashes = new BuildJobStateFileHashes();
                        fileHashes.setCellIndex(cellIndexer.apply(filesystem.getRootPath()));
                        return fileHashes;
                    }
                });
        this.fileHashLoaders = CacheBuilder.newBuilder()
                .build(new CacheLoader<ProjectFilesystem, FileHashLoader>() {
                    @Override
                    public FileHashLoader load(ProjectFilesystem key) throws Exception {
                        return new RecordingFileHashLoader(
                                new StackedFileHashCache(ImmutableList.of(rootCellFileHashCache,
                                        DefaultFileHashCache.createDefaultFileHashCache(key))),
                                key, remoteFileHashes.get(key), new DistBuildConfig(buckConfig));
                    }
                });
        this.ruleKeyFactories = createRuleKeyFactories(sourcePathResolver, ruleFinder, fileHashLoaders, keySeed);
        this.ruleKeys = ruleKeyComputation(actionGraph, this.ruleKeyFactories, executorService);
        this.fileHashes = fileHashesComputation(Futures.transform(this.ruleKeys, Functions.constant(null)),
                this.remoteFileHashes, executorService);
    }

    public static LoadingCache<ProjectFilesystem, DefaultRuleKeyFactory> createRuleKeyFactories(
            final SourcePathResolver sourcePathResolver, final SourcePathRuleFinder ruleFinder,
            final LoadingCache<ProjectFilesystem, ? extends FileHashLoader> fileHashLoaders, final int keySeed) {
        return CacheBuilder.newBuilder().build(new CacheLoader<ProjectFilesystem, DefaultRuleKeyFactory>() {
            @Override
            public DefaultRuleKeyFactory load(ProjectFilesystem key) throws Exception {
                return new DefaultRuleKeyFactory(/* seed */ keySeed, fileHashLoaders.get(key), sourcePathResolver,
                        ruleFinder);
            }
        });
    }

    private static ListenableFuture<ImmutableMap<BuildRule, RuleKey>> ruleKeyComputation(ActionGraph actionGraph,
            final LoadingCache<ProjectFilesystem, DefaultRuleKeyFactory> ruleKeyFactories,
            ListeningExecutorService executorService) {
        List<ListenableFuture<Map.Entry<BuildRule, RuleKey>>> ruleKeyEntries = new ArrayList<>();
        for (final BuildRule rule : actionGraph.getNodes()) {
            ruleKeyEntries.add(executorService.submit(() -> Maps.immutableEntry(rule,
                    ruleKeyFactories.get(rule.getProjectFilesystem()).build(rule))));
        }
        ListenableFuture<List<Map.Entry<BuildRule, RuleKey>>> ruleKeyComputation = Futures
                .allAsList(ruleKeyEntries);
        return Futures.transform(ruleKeyComputation,
                new Function<List<Map.Entry<BuildRule, RuleKey>>, ImmutableMap<BuildRule, RuleKey>>() {
                    @Override
                    public ImmutableMap<BuildRule, RuleKey> apply(List<Map.Entry<BuildRule, RuleKey>> input) {
                        return ImmutableMap.copyOf(input);
                    }
                }, executorService);
    }

    private static ListenableFuture<ImmutableList<BuildJobStateFileHashes>> fileHashesComputation(
            ListenableFuture<Void> ruleKeyComputationForSideEffect,
            final LoadingCache<ProjectFilesystem, BuildJobStateFileHashes> remoteFileHashes,
            ListeningExecutorService executorService) {
        return Futures.transform(ruleKeyComputationForSideEffect,
                new Function<Void, ImmutableList<BuildJobStateFileHashes>>() {
                    @Override
                    public ImmutableList<BuildJobStateFileHashes> apply(Void input) {
                        return ImmutableList.copyOf(remoteFileHashes.asMap().values());
                    }
                }, executorService);
    }

    public List<BuildJobStateFileHashes> getFileHashes() throws IOException, InterruptedException {
        try {
            return fileHashes.get();
        } catch (ExecutionException e) {
            Throwables.propagateIfInstanceOf(e.getCause(), IOException.class);
            Throwables.propagateIfInstanceOf(e.getCause(), InterruptedException.class);
            throw new RuntimeException(e.getCause());
        }
    }

    /**
     * Creates a {@link FileHashCache} that returns the hash codes cached on the remote end.
     *
     * @param projectFilesystem filesystem in which the new cache will be rooted. The serialized state
     *                          only contains relative path, therefore this is needed to indicate
     *                          where on the local machine we wish to transplant the files from the
     *                          remote to.
     * @param remoteFileHashes the serialized state.
     * @return the cache.
     */
    public static FileHashCache createFileHashCache(ProjectFilesystem projectFilesystem,
            BuildJobStateFileHashes remoteFileHashes) {
        return new RemoteStateBasedFileHashCache(projectFilesystem, remoteFileHashes);
    }

    public static ImmutableMap<Path, BuildJobStateFileHashEntry> indexEntriesByPath(
            final ProjectFilesystem projectFilesystem, BuildJobStateFileHashes remoteFileHashes) {
        if (!remoteFileHashes.isSetEntries()) {
            return ImmutableMap.of();
        }
        return FluentIterable.from(remoteFileHashes.entries)
                .filter(input -> !input.isPathIsAbsolute() && !input.isSetArchiveMemberPath())
                .uniqueIndex(input -> projectFilesystem
                        .resolve(MorePaths.pathWithPlatformSeparators(input.getPath().getPath())));
    }

    public static ImmutableMap<ArchiveMemberPath, BuildJobStateFileHashEntry> indexEntriesByArchivePath(
            final ProjectFilesystem projectFilesystem, BuildJobStateFileHashes remoteFileHashes) {
        if (!remoteFileHashes.isSetEntries()) {
            return ImmutableMap.of();
        }
        return FluentIterable.from(remoteFileHashes.entries)
                .filter(input -> !input.isPathIsAbsolute() && input.isSetArchiveMemberPath())
                .uniqueIndex(input -> ArchiveMemberPath.of(
                        projectFilesystem.resolve(MorePaths.pathWithPlatformSeparators(input.getPath().getPath())),
                        Paths.get(input.getArchiveMemberPath())));
    }
}