Java tutorial
/* * Copyright (C) 2014 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.android.builder.internal.compiler; import com.android.annotations.NonNull; import com.android.annotations.Nullable; import com.android.annotations.VisibleForTesting; import com.android.annotations.concurrency.GuardedBy; import com.android.annotations.concurrency.Immutable; import com.android.ide.common.xml.XmlPrettyPrinter; import com.android.sdklib.repository.FullRevision; import com.android.utils.ILogger; import com.android.utils.Pair; import com.android.utils.XmlUtils; import com.google.common.base.Charsets; import com.google.common.base.Objects; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.hash.HashCode; import com.google.common.hash.Hashing; import com.google.common.io.Files; import org.w3c.dom.Attr; import org.w3c.dom.Document; import org.w3c.dom.NamedNodeMap; import org.w3c.dom.Node; import org.w3c.dom.NodeList; import java.io.File; import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.logging.Logger; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; /** */ public abstract class PreProcessCache<T extends PreProcessCache.Key> { private static final String NODE_ITEMS = "items"; private static final String NODE_ITEM = "item"; private static final String NODE_DEX = "dex"; private static final String ATTR_VERSION = "version"; private static final String ATTR_JAR = "jar"; private static final String ATTR_DEX = "dex"; private static final String ATTR_SHA1 = "sha1"; private static final String ATTR_REVISION = "revision"; private static final String XML_VERSION = "2"; protected interface BaseItem { @NonNull File getSourceFile(); @NonNull List<File> getOutputFiles(); @Nullable HashCode getSourceHash(); boolean areOutputFilesPresent(); } /** * Items representing jar/dex files that have been processed during a build. */ @Immutable protected static class Item implements BaseItem { @NonNull private final File mSourceFile; @NonNull private final List<File> mOutputFiles; @NonNull private final CountDownLatch mLatch; Item(@NonNull File sourceFile, @NonNull List<File> outputFiles, @NonNull CountDownLatch latch) { mSourceFile = sourceFile; mOutputFiles = Lists.newArrayList(outputFiles); mLatch = latch; } Item(@NonNull File sourceFile, @NonNull CountDownLatch latch) { mSourceFile = sourceFile; mOutputFiles = Lists.newArrayList(); mLatch = latch; } @Override @NonNull public File getSourceFile() { return mSourceFile; } @Override @NonNull public List<File> getOutputFiles() { return mOutputFiles; } @Nullable @Override public HashCode getSourceHash() { return null; } @NonNull protected CountDownLatch getLatch() { return mLatch; } @Override public boolean areOutputFilesPresent() { boolean filesOk = !mOutputFiles.isEmpty(); for (File outputFile : mOutputFiles) { filesOk &= outputFile.isFile(); } return filesOk; } @Override public String toString() { return "Item{" + "mOutputFiles=" + mOutputFiles + ", mSourceFile=" + mSourceFile + '}'; } } /** * Items representing jar/dex files that have been processed in a previous build, then were * stored in a cache file and then reloaded during the current build. */ @Immutable protected static class StoredItem implements BaseItem { @NonNull private final File mSourceFile; @NonNull private final List<File> mOutputFiles; @NonNull private final HashCode mSourceHash; StoredItem(@NonNull File sourceFile, @NonNull List<File> outputFiles, @NonNull HashCode sourceHash) { mSourceFile = sourceFile; mOutputFiles = Lists.newArrayList(outputFiles); mSourceHash = sourceHash; } @Override @NonNull public File getSourceFile() { return mSourceFile; } @Override @NonNull public List<File> getOutputFiles() { return mOutputFiles; } @Override @NonNull public HashCode getSourceHash() { return mSourceHash; } @Override public boolean areOutputFilesPresent() { boolean filesOk = !mOutputFiles.isEmpty(); for (File outputFile : mOutputFiles) { filesOk &= outputFile.isFile(); } return filesOk; } @Override public String toString() { return "StoredItem{" + "mSourceFile=" + mSourceFile + ", mOutputFiles=" + mOutputFiles + ", mSourceHash=" + mSourceHash + '}'; } } /** * Key to store Item/StoredItem in maps. * The key contains the element that are used for the dex call: * - source file * - build tools revision * - jumbo mode */ @Immutable protected static class Key { @NonNull private final File mSourceFile; @NonNull private final FullRevision mBuildToolsRevision; public static Key of(@NonNull File sourceFile, @NonNull FullRevision buildToolsRevision) { return new Key(sourceFile, buildToolsRevision); } protected Key(@NonNull File sourceFile, @NonNull FullRevision buildToolsRevision) { mSourceFile = sourceFile; mBuildToolsRevision = buildToolsRevision; } @NonNull public FullRevision getBuildToolsRevision() { return mBuildToolsRevision; } @NonNull public File getSourceFile() { return mSourceFile; } @Override public boolean equals(Object o) { if (this == o) { return true; } if (!(o instanceof Key)) { return false; } Key key = (Key) o; if (!mBuildToolsRevision.equals(key.mBuildToolsRevision)) { return false; } if (!mSourceFile.equals(key.mSourceFile)) { return false; } return true; } @Override public int hashCode() { return Objects.hashCode(mSourceFile, mBuildToolsRevision); } } protected interface KeyFactory<T> { T of(@NonNull File sourceFile, @NonNull FullRevision revision, @NonNull NamedNodeMap attrMap); } @GuardedBy("this") private boolean mLoaded = false; @GuardedBy("this") private final Map<T, Item> mMap = Maps.newHashMap(); @GuardedBy("this") private final Map<T, StoredItem> mStoredItems = Maps.newHashMap(); @GuardedBy("this") private int mMisses = 0; @GuardedBy("this") private int mHits = 0; @NonNull protected abstract KeyFactory<T> getKeyFactory(); /** * Loads the stored item. This can be called several times (per subproject), so only * the first call should do something. */ public synchronized void load(@NonNull File itemStorage) { if (mLoaded) { return; } loadItems(itemStorage); mLoaded = true; } /** * Returns an {@link Item} loaded from the cache. If no item can be found this, throws an * exception. * * @param itemKey the key of the item * @return a pair of item, boolean */ protected synchronized Pair<Item, Boolean> getItem(@NonNull T itemKey) { // get the item Item item = mMap.get(itemKey); boolean newItem = false; if (item == null) { // check if we have a stored version. StoredItem storedItem = mStoredItems.get(itemKey); File inputFile = itemKey.getSourceFile(); if (storedItem != null) { // check the sha1 is still valid, and the pre-dex files are still there. if (storedItem.areOutputFilesPresent() && storedItem.getSourceHash().equals(getHash(inputFile))) { Logger.getAnonymousLogger() .info("Cached result for getItem(" + inputFile + "): " + storedItem.getOutputFiles()); for (File f : storedItem.getOutputFiles()) { Logger.getAnonymousLogger() .info(String.format("%s l:%d ts:%d", f, f.length(), f.lastModified())); } // create an item where the outFile is the one stored since it // represent the pre-dexed library already. // Next time this lib needs to be pre-dexed, we'll use the item // rather than the stored item, allowing us to not compute the sha1 again. // Use a 0-count latch since there is nothing to do. item = new Item(inputFile, storedItem.getOutputFiles(), new CountDownLatch(0)); } } // if we didn't find a valid stored item, create a new one. if (item == null) { item = new Item(inputFile, new CountDownLatch(1)); newItem = true; } mMap.put(itemKey, item); } return Pair.of(item, newItem); } @Nullable private static HashCode getHash(@NonNull File file) { try { return Files.hash(file, Hashing.sha1()); } catch (IOException ignored) { } return null; } public synchronized void clear(@Nullable File itemStorage, @Nullable ILogger logger) throws IOException { if (!mMap.isEmpty()) { if (itemStorage != null) { saveItems(itemStorage); } if (logger != null) { logger.info("PREDEX CACHE HITS: " + mHits); logger.info("PREDEX CACHE MISSES: " + mMisses); } } mMap.clear(); mStoredItems.clear(); mHits = 0; mMisses = 0; } private synchronized void loadItems(@NonNull File itemStorage) { if (!itemStorage.isFile()) { return; } try { Document document = XmlUtils.parseUtfXmlFile(itemStorage, true); // get the root node Node rootNode = document.getDocumentElement(); if (rootNode == null || !NODE_ITEMS.equals(rootNode.getLocalName())) { return; } // check the version of the XML NamedNodeMap rootAttrMap = rootNode.getAttributes(); Node versionAttr = rootAttrMap.getNamedItem(ATTR_VERSION); if (versionAttr == null || !XML_VERSION.equals(versionAttr.getNodeValue())) { return; } NodeList nodes = rootNode.getChildNodes(); for (int i = 0, n = nodes.getLength(); i < n; i++) { Node node = nodes.item(i); if (node.getNodeType() != Node.ELEMENT_NODE || !NODE_ITEM.equals(node.getLocalName())) { continue; } NamedNodeMap attrMap = node.getAttributes(); File sourceFile = new File(attrMap.getNamedItem(ATTR_JAR).getNodeValue()); FullRevision revision = FullRevision .parseRevision(attrMap.getNamedItem(ATTR_REVISION).getNodeValue()); List<File> outputFiles = Lists.newArrayList(); NodeList dexNodes = node.getChildNodes(); for (int j = 0, m = dexNodes.getLength(); j < m; j++) { Node dexNode = dexNodes.item(j); if (dexNode.getNodeType() != Node.ELEMENT_NODE || !NODE_DEX.equals(dexNode.getLocalName())) { continue; } NamedNodeMap dexAttrMap = dexNode.getAttributes(); outputFiles.add(new File(dexAttrMap.getNamedItem(ATTR_DEX).getNodeValue())); } StoredItem item = new StoredItem(sourceFile, outputFiles, HashCode.fromString(attrMap.getNamedItem(ATTR_SHA1).getNodeValue())); T key = getKeyFactory().of(sourceFile, revision, attrMap); mStoredItems.put(key, item); } } catch (Exception ignored) { // if we fail to read parts or any of the file, all it'll do is fail to reuse an // already pre-dexed library, so that's not a super big deal. } } protected synchronized void saveItems(@NonNull File itemStorage) throws IOException { // write "compact" blob DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); factory.setNamespaceAware(true); factory.setValidating(false); factory.setIgnoringComments(true); DocumentBuilder builder; try { builder = factory.newDocumentBuilder(); Document document = builder.newDocument(); Node rootNode = document.createElement(NODE_ITEMS); document.appendChild(rootNode); // Set the version Attr attr = document.createAttribute(ATTR_VERSION); attr.setValue(XML_VERSION); rootNode.getAttributes().setNamedItem(attr); Set<T> keys = Sets.newHashSetWithExpectedSize(mMap.size() + mStoredItems.size()); keys.addAll(mMap.keySet()); keys.addAll(mStoredItems.keySet()); for (T key : keys) { Item item = mMap.get(key); if (item != null) { Node itemNode = createItemNode(document, key, item); if (itemNode != null) { rootNode.appendChild(itemNode); } } else { StoredItem storedItem = mStoredItems.get(key); // check that the source file still exists in order to avoid // storing libraries that are gone. if (storedItem != null && storedItem.getSourceFile().isFile() && storedItem.areOutputFilesPresent()) { Node itemNode = createItemNode(document, key, storedItem); if (itemNode != null) { rootNode.appendChild(itemNode); } } } } String content = XmlPrettyPrinter.prettyPrint(document, true); itemStorage.getParentFile().mkdirs(); Files.write(content, itemStorage, Charsets.UTF_8); } catch (ParserConfigurationException e) { } } @Nullable protected Node createItemNode(@NonNull Document document, @NonNull T itemKey, @NonNull BaseItem item) throws IOException { if (!item.areOutputFilesPresent()) { return null; } Node itemNode = document.createElement(NODE_ITEM); Attr attr = document.createAttribute(ATTR_JAR); attr.setValue(item.getSourceFile().getPath()); itemNode.getAttributes().setNamedItem(attr); attr = document.createAttribute(ATTR_REVISION); attr.setValue(itemKey.getBuildToolsRevision().toString()); itemNode.getAttributes().setNamedItem(attr); HashCode hashCode = item.getSourceHash(); if (hashCode == null) { try { hashCode = Files.hash(item.getSourceFile(), Hashing.sha1()); } catch (IOException ex) { // If we can't compute the hash for whatever reason, simply skip this entry. return null; } } attr = document.createAttribute(ATTR_SHA1); attr.setValue(hashCode.toString()); itemNode.getAttributes().setNamedItem(attr); for (File dexFile : item.getOutputFiles()) { Node dexNode = document.createElement(NODE_DEX); itemNode.appendChild(dexNode); attr = document.createAttribute(ATTR_DEX); attr.setValue(dexFile.getPath()); dexNode.getAttributes().setNamedItem(attr); } return itemNode; } protected synchronized void incrementMisses() { mMisses++; } protected synchronized void incrementHits() { mHits++; } @VisibleForTesting /*package*/ synchronized int getMisses() { return mMisses; } @VisibleForTesting /*package*/ synchronized int getHits() { return mHits; } }