Java tutorial
/* * Copyright 2014-present Facebook, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may * not use this file except in compliance with the License. You may obtain * a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations * under the License. */ package com.facebook.buck.android; import static com.facebook.buck.android.SmartDexingStep.DexInputHashesProvider; import com.facebook.buck.dalvik.CanaryFactory; import com.facebook.buck.java.classes.FileLike; import com.facebook.buck.rules.BuildContext; import com.facebook.buck.rules.Sha1HashCode; import com.facebook.buck.step.AbstractExecutionStep; import com.facebook.buck.step.ExecutionContext; import com.facebook.buck.step.Step; import com.facebook.buck.util.HumanReadableException; import com.facebook.buck.util.ProjectFilesystem; import com.google.common.base.Charsets; import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.collect.Collections2; import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Multimap; import com.google.common.hash.Hasher; import com.google.common.hash.Hashing; import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Set; /** * Responsible for bucketing pre-dexed objects into primary and secondary dex files. */ public class PreDexedFilesSorter { private final Optional<DexWithClasses> rDotJavaDex; private final List<DexWithClasses> dexFilesToMerge; private final ClassNameFilter primaryDexFilter; private final long linearAllocHardLimit; private final DexStore dexStore; private final Path secondaryDexJarFilesDir; /** * Directory under the project filesystem where this step may write temporary data. This directory * must exist and be empty before this step writes to it. */ private final Path scratchDirectory; public PreDexedFilesSorter(Optional<DexWithClasses> rDotJavaDex, List<DexWithClasses> dexFilesToMerge, ImmutableSet<String> primaryDexPatterns, Path scratchDirectory, long linearAllocHardLimit, DexStore dexStore, Path secondaryDexJarFilesDir) { this.rDotJavaDex = Preconditions.checkNotNull(rDotJavaDex); this.dexFilesToMerge = Preconditions.checkNotNull(dexFilesToMerge); this.primaryDexFilter = ClassNameFilter.fromConfiguration(Preconditions.checkNotNull(primaryDexPatterns)); this.scratchDirectory = Preconditions.checkNotNull(scratchDirectory); Preconditions.checkState(linearAllocHardLimit > 0); this.linearAllocHardLimit = linearAllocHardLimit; this.dexStore = Preconditions.checkNotNull(dexStore); this.secondaryDexJarFilesDir = Preconditions.checkNotNull(secondaryDexJarFilesDir); } public Result sortIntoPrimaryAndSecondaryDexes(BuildContext context, ImmutableList.Builder<Step> steps) { List<DexWithClasses> primaryDexContents = Lists.newArrayList(); List<List<DexWithClasses>> secondaryDexesContents = Lists.newArrayList(); int primaryDexSize = 0; // R.class files should always be in the primary dex. if (rDotJavaDex.isPresent()) { primaryDexSize += rDotJavaDex.get().getSizeEstimate(); primaryDexContents.add(rDotJavaDex.get()); } // Sort dex files so that there's a better chance of the same set of pre-dexed files to end up // in a given secondary dex file. ImmutableList<DexWithClasses> sortedDexFilesToMerge = FluentIterable.from(dexFilesToMerge) .toSortedList(DexWithClasses.DEX_WITH_CLASSES_COMPARATOR); // Bucket each DexWithClasses into the appropriate dex file. List<DexWithClasses> currentSecondaryDexContents = null; int currentSecondaryDexSize = 0; for (DexWithClasses dexWithClasses : sortedDexFilesToMerge) { if (mustBeInPrimaryDex(dexWithClasses)) { // Case 1: Entry must be in the primary dex. primaryDexSize += dexWithClasses.getSizeEstimate(); if (primaryDexSize > linearAllocHardLimit) { context.logError( "DexWithClasses %s with cost %s puts the linear alloc estimate for the primary dex " + "at %s, exceeding the maximum of %s.", dexWithClasses.getPathToDexFile(), dexWithClasses.getSizeEstimate(), primaryDexSize, linearAllocHardLimit); throw new HumanReadableException("Primary dex exceeds linear alloc limit."); } primaryDexContents.add(dexWithClasses); } else { // Case 2: Entry must go in a secondary dex. // If the individual DexWithClasses exceeds the limit for a secondary dex, then we have done // something horribly wrong. if (dexWithClasses.getSizeEstimate() > linearAllocHardLimit) { context.logError( "DexWithClasses %s with cost %s exceeds the max cost %s for a secondary dex file.", dexWithClasses.getPathToDexFile(), dexWithClasses.getSizeEstimate(), linearAllocHardLimit); throw new HumanReadableException("Secondary dex exceeds linear alloc limit."); } // If there is no current secondary dex, or dexWithClasses would put the current secondary // dex over the cost threshold, then create a new secondary dex and initialize it with a // canary. if (currentSecondaryDexContents == null || dexWithClasses.getSizeEstimate() + currentSecondaryDexSize > linearAllocHardLimit) { DexWithClasses canary = createCanary(secondaryDexesContents.size() + 1, steps); currentSecondaryDexContents = Lists.newArrayList(canary); currentSecondaryDexSize = canary.getSizeEstimate(); secondaryDexesContents.add(currentSecondaryDexContents); } // Now add the contributions from the dexWithClasses entry. currentSecondaryDexContents.add(dexWithClasses); currentSecondaryDexSize += dexWithClasses.getSizeEstimate(); } } ImmutableSet<Path> primaryDexInputs = FluentIterable.from(primaryDexContents) .transform(DexWithClasses.TO_PATH).toSet(); Map<Path, DexWithClasses> metadataTxtEntries = Maps.newHashMap(); String pattern = "secondary-%d" + dexStore.getExtension(); ImmutableMultimap.Builder<Path, Path> secondaryOutputToInputs = ImmutableMultimap.builder(); for (int index = 0; index < secondaryDexesContents.size(); index++) { String secondaryDexFilename = String.format(pattern, index + 1); Path pathToSecondaryDex = secondaryDexJarFilesDir.resolve(secondaryDexFilename); metadataTxtEntries.put(pathToSecondaryDex, secondaryDexesContents.get(index).get(0)); Collection<Path> dexContentPaths = Collections2.transform(secondaryDexesContents.get(index), DexWithClasses.TO_PATH); secondaryOutputToInputs.putAll(pathToSecondaryDex, dexContentPaths); } return new Result(primaryDexInputs, secondaryOutputToInputs.build(), metadataTxtEntries, getDexInputsHashes(primaryDexContents, secondaryDexesContents)); } private static ImmutableMap<Path, Sha1HashCode> getDexInputsHashes(List<DexWithClasses> primaryDexContents, List<List<DexWithClasses>> secondaryDexesContents) { Iterable<DexWithClasses> allInputs = Iterables.concat(primaryDexContents, Iterables.concat(secondaryDexesContents)); ImmutableMap.Builder<Path, Sha1HashCode> dexInputsHashes = ImmutableMap.builder(); for (DexWithClasses dexWithClasses : allInputs) { dexInputsHashes.put(dexWithClasses.getPathToDexFile(), dexWithClasses.getClassesHash()); } return dexInputsHashes.build(); } private boolean mustBeInPrimaryDex(DexWithClasses dexWithClasses) { for (String className : dexWithClasses.getClassNames()) { if (primaryDexFilter.matches(className)) { return true; } } return false; } /** * @see com.facebook.buck.dalvik.CanaryFactory#create(int) */ private DexWithClasses createCanary(final int index, ImmutableList.Builder<Step> steps) { final FileLike fileLike = CanaryFactory.create(index); final String canaryDirName = "canary_" + String.valueOf(index); final Path scratchDirectoryForCanaryClass = scratchDirectory.resolve(canaryDirName); // Strip the .class suffix to get the class name for the DexWithClasses object. final String relativePathToClassFile = fileLike.getRelativePath(); Preconditions.checkState(relativePathToClassFile.endsWith(".class")); final String className = relativePathToClassFile.replaceFirst("\\.class$", ""); // Write out the .class file. steps.add(new AbstractExecutionStep("write_canary_class") { @Override public int execute(ExecutionContext context) { Path classFile = scratchDirectoryForCanaryClass.resolve(relativePathToClassFile); ProjectFilesystem projectFilesystem = context.getProjectFilesystem(); try (InputStream inputStream = fileLike.getInput()) { projectFilesystem.createParentDirs(classFile); projectFilesystem.copyToPath(inputStream, classFile); } catch (IOException e) { context.logError(e, "Error writing canary class file: %s.", classFile.toString()); return 1; } return 0; } }); return new DexWithClasses() { @Override public int getSizeEstimate() { // Because we do not know the units being used for DEX size estimation and the canary should // be very small, assume the size is zero. return 0; } @Override public Path getPathToDexFile() { return scratchDirectoryForCanaryClass; } @Override public ImmutableSet<String> getClassNames() { return ImmutableSet.of(className); } @Override public Sha1HashCode getClassesHash() { // The only thing unique to canary classes is the index, which is captured by canaryDirName. Hasher hasher = Hashing.sha1().newHasher(); hasher.putString(canaryDirName, Charsets.UTF_8); return new Sha1HashCode(hasher.hash().toString()); } }; } public static class Result { public final Set<Path> primaryDexInputs; public final Multimap<Path, Path> secondaryOutputToInputs; public final Map<Path, DexWithClasses> metadataTxtDexEntries; public final DexInputHashesProvider dexInputHashesProvider; public Result(Set<Path> primaryDexInputs, Multimap<Path, Path> secondaryOutputToInputs, Map<Path, DexWithClasses> metadataTxtDexEntries, final ImmutableMap<Path, Sha1HashCode> dexInputHashes) { this.primaryDexInputs = Preconditions.checkNotNull(primaryDexInputs); this.secondaryOutputToInputs = Preconditions.checkNotNull(secondaryOutputToInputs); this.metadataTxtDexEntries = Preconditions.checkNotNull(metadataTxtDexEntries); Preconditions.checkNotNull(dexInputHashes); this.dexInputHashesProvider = new DexInputHashesProvider() { @Override public ImmutableMap<Path, Sha1HashCode> getDexInputHashes() { return dexInputHashes; } }; } } }