List of usage examples for com.google.common.collect ImmutableMap keySet
public ImmutableSet<K> keySet()
From source file:com.facebook.buck.android.NonPreDexedDexBuildable.java
/** @return the resulting set of ProGuarded classpath entries to dex. */ @VisibleForTesting// www.ja va 2s . com ImmutableSet<Path> addProguardCommands(Set<Path> classpathEntriesToDex, Set<Path> depsProguardConfigs, boolean skipProguard, ImmutableList.Builder<Step> steps, BuildableContext buildableContext, BuildContext buildContext) { // Create list of proguard Configs for the app project and its dependencies ImmutableSet.Builder<Path> proguardConfigsBuilder = ImmutableSet.builder(); proguardConfigsBuilder.addAll(depsProguardConfigs); if (proguardConfig.isPresent()) { proguardConfigsBuilder.add(buildContext.getSourcePathResolver().getAbsolutePath(proguardConfig.get())); } for (SourcePath aaptGeneratedProguardConfigFile : proguardConfigs) { proguardConfigsBuilder .add(buildContext.getSourcePathResolver().getAbsolutePath(aaptGeneratedProguardConfigFile)); } // Transform our input classpath to a set of output locations for each input classpath. // TODO(devjasta): the output path we choose is the result of a slicing function against // input classpath. This is fragile and should be replaced with knowledge of the BuildTarget. ImmutableMap<Path, Path> inputOutputEntries = classpathEntriesToDex.stream().collect( ImmutableMap.toImmutableMap(java.util.function.Function.identity(), (path) -> AndroidBinaryBuildable .getProguardOutputFromInputClasspath(getProguardInputsDir(), path))); // Run ProGuard on the classpath entries. ProGuardObfuscateStep.create(androidPlatformTarget, javaRuntimeLauncher.getCommandPrefix(buildContext.getSourcePathResolver()), getProjectFilesystem(), proguardJarOverride.isPresent() ? Optional .of(buildContext.getSourcePathResolver().getAbsolutePath(proguardJarOverride.get())) : Optional.empty(), proguardMaxHeapSize, proguardAgentPath, proguardConfigsBuilder.build(), sdkProguardConfig, optimizationPasses, proguardJvmArgs, inputOutputEntries, buildContext.getSourcePathResolver().getAllAbsolutePaths(additionalJarsForProguardAndDesugar), getProguardConfigDir(), buildableContext, buildContext, skipProguard, steps); // Apply the transformed inputs to the classpath (this will modify deps.classpathEntriesToDex // so that we're now dexing the proguarded artifacts). However, if we are not running // ProGuard then return the input classes to dex. if (skipProguard) { return ImmutableSet.copyOf(inputOutputEntries.keySet()); } else { return ImmutableSet.copyOf(inputOutputEntries.values()); } }
From source file:org.zanata.client.commands.push.RawPushCommand.java
/** * Return map of validated DocumentType to set of corresponding extensions, by applying the user's * file type options to the server's accepted file types. * * Validate user input file types against server accepted file types * * Valid input - properties_utf8,properties[txt],plain_text[md;markdown] * * - Each file type must appear only once - e.g. * - valid: "html,properties,txt"//from ww w . jav a2 s .com * - invalid: "html,properties,html" * - Same file extension must not appear in multiple file types - e.g. plain_text[txt],properties[txt] * @param serverFileTypes * @param fileTypesSpec */ public ImmutableList<FileTypeInfo> getActualFileTypes(List<FileTypeInfo> serverFileTypes, List<String> fileTypesSpec) { // cumulative list of activated types ImmutableList.Builder<FileTypeInfo> docTypeMappings = new ImmutableList.Builder<>(); // types which have been specified by the user so far Set<FileTypeName> seenUserDocTypes = new HashSet<>(); // extensions which have been specified by the user so far Set<String> seenUserExtensions = new HashSet<>(); if (fileTypesSpec.isEmpty()) { return ImmutableList.of(); } for (String fileTypeSpec : fileTypesSpec) { @Nullable FileTypeName userType = extractFileTypeName(fileTypeSpec); ImmutableMap<String, String> userExtensions; if (userType == null) { // try parameter as a list of file extensions: ZNTA-1248 String[] exts = fileTypeSpec.split(","); ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>(); for (String ext : exts) { builder.put(ext, ext); } userExtensions = builder.build(); } else { userExtensions = extractExtensions(fileTypeSpec); } validateFileExtensions(userType, userExtensions, serverFileTypes); assert userType != null; @Nullable FileTypeInfo fileTypeInfo = serverFileTypes.stream() .filter((FileTypeInfo info) -> info.getType().equals(userType)).findAny().orElse(null); // throw error if file type is not supported by server if (fileTypeInfo == null) { String msg = Messages.format("file.type.typeNotSupported", userType); throw new InvalidUserInputException(msg); } if (!seenUserDocTypes.add(userType)) { //throw error if file type is listed more than once String msg = Messages.format("file.type.duplicateFileType", userType); log.error(msg); throw new RuntimeException(msg); } for (String srcExt : userExtensions.keySet()) { //throw error if same file extension found in multiple file types if (!seenUserExtensions.add(srcExt)) { String msg = Messages.format("file.type.conflictExtension", srcExt, userType); log.error(msg); throw new RuntimeException(msg); } } // Use the extensions from docTypeMappingSpec if specified, // otherwise use the extensions from server. Map<String, String> filteredExtensions = userExtensions.isEmpty() ? fileTypeInfo.getExtensions() : userExtensions; docTypeMappings.add(new FileTypeInfo(userType, filteredExtensions)); } return docTypeMappings.build(); }
From source file:com.android.builder.shrinker.Shrinker.java
public void handleFileChanges(Collection<TransformInput> inputs, @NonNull TransformOutputProvider output, final ImmutableMap<ShrinkType, KeepRules> keepRules) throws IOException { mGraph.loadState();/*from w ww .ja v a 2 s . c o m*/ final ImmutableMap<ShrinkType, Set<T>> modifiedClasses = buildMapPerShrinkType(keepRules); for (final TransformInput input : inputs) { for (JarInput jarInput : input.getJarInputs()) { switch (jarInput.getStatus()) { case ADDED: break; case REMOVED: break; case CHANGED: break; } } for (DirectoryInput directoryInput : input.getDirectoryInputs()) { for (final Map.Entry<File, Status> changedFile : directoryInput.getChangedFiles().entrySet()) { mExecutor.execute(new Callable<Void>() { @Override public Void call() throws Exception { switch (changedFile.getValue()) { case ADDED: throw todo("new file added"); case REMOVED: throw todo("removed file"); case CHANGED: processChangedClassFile(changedFile.getKey(), keepRules, modifiedClasses); break; } return null; } }); } } } waitForAllTasks(); for (ShrinkType shrinkType : keepRules.keySet()) { updateClassFiles(modifiedClasses.get(shrinkType), inputs, output); } waitForAllTasks(); }
From source file:com.facebook.buck.apple.ProjectGenerator.java
private void generateBuildWithBuckTarget(TargetNode<?> targetNode) throws IOException { final BuildTarget buildTarget = targetNode.getBuildTarget(); String buckTargetProductName = getXcodeTargetName(buildTarget) + BUILD_WITH_BUCK_POSTFIX; PBXAggregateTarget buildWithBuckTarget = new PBXAggregateTarget(buckTargetProductName); buildWithBuckTarget.setProductName(buckTargetProductName); PBXShellScriptBuildPhase buildShellScriptBuildPhase = new PBXShellScriptBuildPhase(); buildShellScriptBuildPhase.setShellScript(getBuildWithBuckShellScript(targetNode)); buildWithBuckTarget.getBuildPhases().add(buildShellScriptBuildPhase); // Only add a shell script for fixing UUIDs if it is an AppleBundle if (targetNode.getType().equals(AppleBundleDescription.TYPE)) { PBXShellScriptBuildPhase fixUUIDShellScriptBuildPhase = new PBXShellScriptBuildPhase(); fixUUIDShellScriptBuildPhase.setShellScript(getFixUUIDShellScript(targetNode)); buildWithBuckTarget.getBuildPhases().add(fixUUIDShellScriptBuildPhase); PBXShellScriptBuildPhase codesignPhase = new PBXShellScriptBuildPhase(); codesignPhase.setShellScript(getCodesignShellScript(targetNode)); buildWithBuckTarget.getBuildPhases().add(codesignPhase); }/*from w w w .j av a2 s . com*/ TargetNode<CxxLibraryDescription.Arg> node = getAppleNativeNode(targetGraph, targetNode).get(); ImmutableMap<String, ImmutableMap<String, String>> configs = getXcodeBuildConfigurationsForTargetNode(node, ImmutableMap.<String, String>of()).get(); XCConfigurationList configurationList = new XCConfigurationList(); PBXGroup group = project.getMainGroup() .getOrCreateDescendantGroupByPath(FluentIterable.from(buildTarget.getBasePath()) .transform(Functions.toStringFunction()).toList()) .getOrCreateChildGroupByName(getXcodeTargetName(buildTarget)); for (String configurationName : configs.keySet()) { XCBuildConfiguration configuration = configurationList.getBuildConfigurationsByName() .getUnchecked(configurationName); configuration.setBaseConfigurationReference(getConfigurationFileReference(group, getConfigurationNameToXcconfigPath(buildTarget).apply(configurationName))); NSDictionary inlineSettings = new NSDictionary(); inlineSettings.put("HEADER_SEARCH_PATHS", ""); inlineSettings.put("LIBRARY_SEARCH_PATHS", ""); inlineSettings.put("FRAMEWORK_SEARCH_PATHS", ""); configuration.setBuildSettings(inlineSettings); } buildWithBuckTarget.setBuildConfigurationList(configurationList); project.getTargets().add(buildWithBuckTarget); targetNodeToGeneratedProjectTargetBuilder.put(targetNode, buildWithBuckTarget); }
From source file:com.facebook.buck.apple.project_generator.ProjectGenerator.java
private void generateBuildWithBuckTarget(TargetNode<?, ?> targetNode) { final BuildTarget buildTarget = targetNode.getBuildTarget(); String buckTargetProductName = getXcodeTargetName(buildTarget) + BUILD_WITH_BUCK_POSTFIX; PBXAggregateTarget buildWithBuckTarget = new PBXAggregateTarget(buckTargetProductName); buildWithBuckTarget.setProductName(buckTargetProductName); PBXShellScriptBuildPhase buildShellScriptBuildPhase = new PBXShellScriptBuildPhase(); buildShellScriptBuildPhase.setShellScript(getBuildWithBuckShellScript(targetNode)); buildWithBuckTarget.getBuildPhases().add(buildShellScriptBuildPhase); // Only add a shell script for fixing UUIDs if it is an AppleBundle if (targetNode.getDescription() instanceof AppleBundleDescription) { PBXShellScriptBuildPhase codesignPhase = new PBXShellScriptBuildPhase(); codesignPhase.setShellScript(getCodesignShellScript(targetNode)); buildWithBuckTarget.getBuildPhases().add(codesignPhase); }/* w w w . j a v a2 s .com*/ TargetNode<CxxLibraryDescription.Arg, ?> node = getAppleNativeNode(targetGraph, targetNode).get(); ImmutableMap<String, ImmutableMap<String, String>> configs = getXcodeBuildConfigurationsForTargetNode(node, ImmutableMap.of()).get(); XCConfigurationList configurationList = new XCConfigurationList(); PBXGroup group = project.getMainGroup() .getOrCreateDescendantGroupByPath( StreamSupport.stream(buildTarget.getBasePath().spliterator(), false).map(Object::toString) .collect(MoreCollectors.toImmutableList())) .getOrCreateChildGroupByName(getXcodeTargetName(buildTarget)); for (String configurationName : configs.keySet()) { XCBuildConfiguration configuration = configurationList.getBuildConfigurationsByName() .getUnchecked(configurationName); configuration.setBaseConfigurationReference(getConfigurationFileReference(group, getConfigurationNameToXcconfigPath(buildTarget).apply(configurationName))); NSDictionary inlineSettings = new NSDictionary(); inlineSettings.put("HEADER_SEARCH_PATHS", ""); inlineSettings.put("LIBRARY_SEARCH_PATHS", ""); inlineSettings.put("FRAMEWORK_SEARCH_PATHS", ""); configuration.setBuildSettings(inlineSettings); } buildWithBuckTarget.setBuildConfigurationList(configurationList); project.getTargets().add(buildWithBuckTarget); targetNodeToGeneratedProjectTargetBuilder.put(targetNode, buildWithBuckTarget); }
From source file:com.facebook.buck.cxx.CxxDescriptionEnhancer.java
public static CxxLinkAndCompileRules createBuildRulesForCxxBinary(BuildRuleParams params, BuildRuleResolver resolver, CxxBuckConfig cxxBuckConfig, CxxPlatform cxxPlatform, ImmutableMap<String, CxxSource> srcs, ImmutableMap<Path, SourcePath> headers, Iterable<? extends BuildRule> deps, Optional<StripStyle> stripStyle, Optional<LinkerMapMode> flavoredLinkerMapMode, Linker.LinkableDepType linkStyle, ImmutableList<String> preprocessorFlags, PatternMatchedCollection<ImmutableList<String>> platformPreprocessorFlags, ImmutableMap<CxxSource.Type, ImmutableList<String>> langPreprocessorFlags, ImmutableSortedSet<FrameworkPath> frameworks, ImmutableSortedSet<FrameworkPath> libraries, ImmutableList<String> compilerFlags, ImmutableMap<CxxSource.Type, ImmutableList<String>> langCompilerFlags, PatternMatchedCollection<ImmutableList<String>> platformCompilerFlags, Optional<SourcePath> prefixHeader, Optional<SourcePath> precompiledHeader, ImmutableList<String> linkerFlags, PatternMatchedCollection<ImmutableList<String>> platformLinkerFlags, Optional<Linker.CxxRuntimeType> cxxRuntimeType, ImmutableList<String> includeDirs, Optional<Boolean> xcodePrivateHeadersSymlinks) throws NoSuchBuildTargetException { SourcePathRuleFinder ruleFinder = new SourcePathRuleFinder(resolver); SourcePathResolver sourcePathResolver = new SourcePathResolver(ruleFinder); // TODO(beefon): should be: // Path linkOutput = getLinkOutputPath( // createCxxLinkTarget(params.getBuildTarget(), flavoredLinkerMapMode), // params.getProjectFilesystem()); BuildTarget target = params.getBuildTarget(); if (flavoredLinkerMapMode.isPresent()) { target = target.withAppendedFlavors(flavoredLinkerMapMode.get().getFlavor()); }//from ww w . ja va 2s. c om Path linkOutput = getBinaryOutputPath(target, params.getProjectFilesystem(), cxxPlatform.getBinaryExtension()); ImmutableList.Builder<Arg> argsBuilder = ImmutableList.builder(); CommandTool.Builder executableBuilder = new CommandTool.Builder(); // Setup the header symlink tree and combine all the preprocessor input from this rule // and all dependencies. boolean shouldCreatePrivateHeadersSymlinks = xcodePrivateHeadersSymlinks.orElse(true); HeaderSymlinkTree headerSymlinkTree = requireHeaderSymlinkTree(params, resolver, sourcePathResolver, cxxPlatform, headers, HeaderVisibility.PRIVATE, shouldCreatePrivateHeadersSymlinks); Optional<SymlinkTree> sandboxTree = Optional.empty(); if (cxxBuckConfig.sandboxSources()) { sandboxTree = createSandboxTree(params, resolver, cxxPlatform); } ImmutableList<CxxPreprocessorInput> cxxPreprocessorInput = collectCxxPreprocessorInput(params, cxxPlatform, CxxFlags.getLanguageFlags( preprocessorFlags, platformPreprocessorFlags, langPreprocessorFlags, cxxPlatform), ImmutableList.of(headerSymlinkTree), frameworks, CxxPreprocessables.getTransitiveCxxPreprocessorInput(cxxPlatform, RichStream.from(deps).filter(CxxPreprocessorDep.class::isInstance).toImmutableList()), includeDirs, sandboxTree); // Generate and add all the build rules to preprocess and compile the source to the // resolver and get the `SourcePath`s representing the generated object files. ImmutableMap<CxxPreprocessAndCompile, SourcePath> objects = CxxSourceRuleFactory .requirePreprocessAndCompileRules(params, resolver, sourcePathResolver, ruleFinder, cxxBuckConfig, cxxPlatform, cxxPreprocessorInput, CxxFlags.getLanguageFlags(compilerFlags, platformCompilerFlags, langCompilerFlags, cxxPlatform), prefixHeader, precompiledHeader, srcs, linkStyle == Linker.LinkableDepType.STATIC ? CxxSourceRuleFactory.PicType.PDC : CxxSourceRuleFactory.PicType.PIC, sandboxTree); // Build up the linker flags, which support macro expansion. ImmutableList<String> resolvedLinkerFlags = CxxFlags.getFlags(linkerFlags, platformLinkerFlags, cxxPlatform); argsBuilder.addAll(resolvedLinkerFlags.stream().map(MacroArg.toMacroArgFunction(MACRO_HANDLER, params.getBuildTarget(), params.getCellRoots(), resolver)::apply).iterator()); // Special handling for dynamically linked binaries. if (linkStyle == Linker.LinkableDepType.SHARED) { // Create a symlink tree with for all shared libraries needed by this binary. SymlinkTree sharedLibraries = requireSharedLibrarySymlinkTree(params, resolver, sourcePathResolver, cxxPlatform, deps, NativeLinkable.class::isInstance); // Embed a origin-relative library path into the binary so it can find the shared libraries. // The shared libraries root is absolute. Also need an absolute path to the linkOutput Path absLinkOut = params.getBuildTarget().getCellPath().resolve(linkOutput); argsBuilder.addAll(StringArg.from(Linkers.iXlinker("-rpath", String.format("%s/%s", cxxPlatform.getLd().resolve(resolver).origin(), absLinkOut.getParent().relativize(sharedLibraries.getRoot()).toString())))); // Add all the shared libraries and the symlink tree as inputs to the tool that represents // this binary, so that users can attach the proper deps. executableBuilder.addDep(sharedLibraries); executableBuilder.addInputs(sharedLibraries.getLinks().values()); } // Add object files into the args. ImmutableList<SourcePathArg> objectArgs = SourcePathArg.from(sourcePathResolver, objects.values()).stream() .map(input -> { Preconditions.checkArgument(input instanceof SourcePathArg); return (SourcePathArg) input; }).collect(MoreCollectors.toImmutableList()); argsBuilder.addAll(FileListableLinkerInputArg.from(objectArgs)); BuildTarget linkRuleTarget = createCxxLinkTarget(params.getBuildTarget(), flavoredLinkerMapMode); CxxLink cxxLink = createCxxLinkRule(params, resolver, cxxBuckConfig, cxxPlatform, RichStream.from(deps).filter(NativeLinkable.class).toImmutableList(), linkStyle, frameworks, libraries, cxxRuntimeType, sourcePathResolver, ruleFinder, linkOutput, argsBuilder, linkRuleTarget); BuildRule binaryRuleForExecutable; Optional<CxxStrip> cxxStrip = Optional.empty(); if (stripStyle.isPresent()) { BuildRuleParams cxxParams = params; if (flavoredLinkerMapMode.isPresent()) { cxxParams = params.withFlavor(flavoredLinkerMapMode.get().getFlavor()); } CxxStrip stripRule = createCxxStripRule(cxxParams, resolver, stripStyle.get(), sourcePathResolver, cxxLink, cxxPlatform); cxxStrip = Optional.of(stripRule); binaryRuleForExecutable = stripRule; } else { binaryRuleForExecutable = cxxLink; } // Add the output of the link as the lone argument needed to invoke this binary as a tool. executableBuilder.addArg(new SourcePathArg(sourcePathResolver, new BuildTargetSourcePath(binaryRuleForExecutable.getBuildTarget()))); return new CxxLinkAndCompileRules(cxxLink, cxxStrip, ImmutableSortedSet.copyOf(objects.keySet()), executableBuilder.build()); }
From source file:com.android.build.gradle.tasks.PackageAndroidArtifact.java
/** * Packages the application incrementally. In case of instant run packaging, this is not a * perfectly incremental task as some files are always rewritten even if no change has * occurred./*from w w w . ja v a 2s. co m*/ * * @param changedDex incremental dex packaging data * @param changedJavaResources incremental java resources * @param changedAssets incremental assets * @param changedAndroidResources incremental Android resource * @param changedNLibs incremental native libraries changed * @throws IOException failed to package the APK */ private void doTask(@NonNull ImmutableMap<RelativeFile, FileStatus> changedDex, @NonNull ImmutableMap<RelativeFile, FileStatus> changedJavaResources, @NonNull ImmutableMap<RelativeFile, FileStatus> changedAssets, @NonNull ImmutableMap<RelativeFile, FileStatus> changedAndroidResources, @NonNull ImmutableMap<RelativeFile, FileStatus> changedNLibs) throws IOException { ImmutableMap.Builder<RelativeFile, FileStatus> javaResourcesForApk = ImmutableMap.builder(); javaResourcesForApk.putAll(changedJavaResources); Collection<File> instantRunDexBaseFiles; switch (dexPackagingPolicy) { case INSTANT_RUN_SHARDS_IN_SINGLE_APK: /* * If we're doing instant run, then we don't want to treat all dex archives * as dex archives for packaging. We will package some of the dex files as * resources. * * All dex files in directories whose name contains INSTANT_RUN_PACKAGES_PREFIX * are kept in the apk as dex files. All other dex files are placed as * resources as defined by makeInstantRunResourcesFromDex. */ instantRunDexBaseFiles = getDexFolders().stream() .filter(input -> input.getName().contains(INSTANT_RUN_PACKAGES_PREFIX)) .collect(Collectors.toSet()); Iterable<File> nonInstantRunDexBaseFiles = getDexFolders().stream() .filter(f -> !instantRunDexBaseFiles.contains(f)).collect(Collectors.toSet()); ImmutableMap<RelativeFile, FileStatus> newInstantRunResources = makeInstantRunResourcesFromDex( nonInstantRunDexBaseFiles); @SuppressWarnings("unchecked") ImmutableMap<RelativeFile, FileStatus> updatedChangedResources = IncrementalRelativeFileSets .union(Sets.newHashSet(changedJavaResources, newInstantRunResources)); changedJavaResources = updatedChangedResources; changedDex = ImmutableMap.copyOf(Maps.filterKeys(changedDex, Predicates.compose(Predicates.in(instantRunDexBaseFiles), RelativeFile.EXTRACT_BASE))); break; case INSTANT_RUN_MULTI_APK: instantRunDexBaseFiles = getDexFolders().stream() .filter(input -> input.getName().contains(InstantRunSlicer.MAIN_SLICE_NAME)) .collect(Collectors.toSet()); changedDex = ImmutableMap.copyOf(Maps.filterKeys(changedDex, Predicates.compose(Predicates.in(instantRunDexBaseFiles), RelativeFile.EXTRACT_BASE))); case STANDARD: break; default: throw new RuntimeException("Unhandled DexPackagingPolicy : " + getDexPackagingPolicy()); } PrivateKey key; X509Certificate certificate; boolean v1SigningEnabled; boolean v2SigningEnabled; try { if (signingConfig != null && signingConfig.isSigningReady()) { CertificateInfo certificateInfo = KeystoreHelper.getCertificateInfo(signingConfig.getStoreType(), checkNotNull(signingConfig.getStoreFile()), checkNotNull(signingConfig.getStorePassword()), checkNotNull(signingConfig.getKeyPassword()), checkNotNull(signingConfig.getKeyAlias())); key = certificateInfo.getKey(); certificate = certificateInfo.getCertificate(); v1SigningEnabled = signingConfig.isV1SigningEnabled(); v2SigningEnabled = signingConfig.isV2SigningEnabled(); } else { key = null; certificate = null; v1SigningEnabled = false; v2SigningEnabled = false; } ApkCreatorFactory.CreationData creationData = new ApkCreatorFactory.CreationData(getOutputFile(), key, certificate, v1SigningEnabled, v2SigningEnabled, null, // BuiltBy getBuilder().getCreatedBy(), getMinSdkVersion(), PackagingUtils.getNativeLibrariesLibrariesPackagingMode(manifest), getNoCompressPredicate()::apply); try (IncrementalPackager packager = createPackager(creationData)) { packager.updateDex(changedDex); packager.updateJavaResources(changedJavaResources); packager.updateAssets(changedAssets); packager.updateAndroidResources(changedAndroidResources); packager.updateNativeLibraries(changedNLibs); } } catch (PackagerException | KeytoolException e) { throw new RuntimeException(e); } /* * Save all used zips in the cache. */ Stream.concat(changedDex.keySet().stream(), Stream.concat(changedJavaResources.keySet().stream(), Stream.concat(changedAndroidResources.keySet().stream(), changedNLibs.keySet().stream()))) .map(RelativeFile::getBase).filter(File::isFile).distinct().forEach((File f) -> { try { cacheByPath.add(f); } catch (IOException e) { throw new IOExceptionWrapper(e); } }); // Mark this APK production, this will eventually be saved when instant-run is enabled. // this might get overridden if the apk is signed/aligned. try { instantRunContext.addChangedFile(instantRunFileType, getOutputFile()); } catch (IOException e) { throw new BuildException(e.getMessage(), e); } }
From source file:org.elasticsearch.index.gateway.blobstore.BlobStoreIndexShardGateway.java
private void doSnapshot(final Snapshot snapshot) throws IndexShardGatewaySnapshotFailedException { ImmutableMap<String, BlobMetaData> blobs; try {//from w w w . ja va2 s. c o m blobs = blobContainer.listBlobs(); } catch (IOException e) { throw new IndexShardGatewaySnapshotFailedException(shardId, "failed to list blobs", e); } long generation = findLatestFileNameGeneration(blobs); CommitPoints commitPoints = buildCommitPoints(blobs); currentSnapshotStatus.index().startTime(System.currentTimeMillis()); currentSnapshotStatus.updateStage(SnapshotStatus.Stage.INDEX); final SnapshotIndexCommit snapshotIndexCommit = snapshot.indexCommit(); final Translog.Snapshot translogSnapshot = snapshot.translogSnapshot(); final CountDownLatch indexLatch = new CountDownLatch(snapshotIndexCommit.getFiles().length); final CopyOnWriteArrayList<Throwable> failures = new CopyOnWriteArrayList<Throwable>(); final List<CommitPoint.FileInfo> indexCommitPointFiles = Lists.newArrayList(); int indexNumberOfFiles = 0; long indexTotalFilesSize = 0; for (final String fileName : snapshotIndexCommit.getFiles()) { StoreFileMetaData md; try { md = store.metaData(fileName); } catch (IOException e) { throw new IndexShardGatewaySnapshotFailedException(shardId, "Failed to get store file metadata", e); } boolean snapshotRequired = false; if (snapshot.indexChanged() && fileName.equals(snapshotIndexCommit.getSegmentsFileName())) { snapshotRequired = true; // we want to always snapshot the segment file if the index changed } CommitPoint.FileInfo fileInfo = commitPoints.findPhysicalIndexFile(fileName); if (fileInfo == null || !fileInfo.isSame(md) || !commitPointFileExistsInBlobs(fileInfo, blobs)) { // commit point file does not exists in any commit point, or has different length, or does not fully exists in the listed blobs snapshotRequired = true; } if (snapshotRequired) { indexNumberOfFiles++; indexTotalFilesSize += md.length(); // create a new FileInfo try { CommitPoint.FileInfo snapshotFileInfo = new CommitPoint.FileInfo( fileNameFromGeneration(++generation), fileName, md.length(), md.checksum()); indexCommitPointFiles.add(snapshotFileInfo); snapshotFile(snapshotIndexCommit.getDirectory(), snapshotFileInfo, indexLatch, failures); } catch (IOException e) { failures.add(e); indexLatch.countDown(); } } else { indexCommitPointFiles.add(fileInfo); indexLatch.countDown(); } } currentSnapshotStatus.index().files(indexNumberOfFiles, indexTotalFilesSize); try { indexLatch.await(); } catch (InterruptedException e) { failures.add(e); } if (!failures.isEmpty()) { throw new IndexShardGatewaySnapshotFailedException(shardId(), "Failed to perform snapshot (index files)", failures.get(failures.size() - 1)); } currentSnapshotStatus.index().time(System.currentTimeMillis() - currentSnapshotStatus.index().startTime()); currentSnapshotStatus.updateStage(SnapshotStatus.Stage.TRANSLOG); currentSnapshotStatus.translog().startTime(System.currentTimeMillis()); // Note, we assume the snapshot is always started from "base 0". We need to seek forward if we want to lastTranslogPosition if we want the delta List<CommitPoint.FileInfo> translogCommitPointFiles = Lists.newArrayList(); int expectedNumberOfOperations = 0; boolean snapshotRequired = false; if (snapshot.newTranslogCreated()) { if (translogSnapshot.lengthInBytes() > 0) { snapshotRequired = true; expectedNumberOfOperations = translogSnapshot.estimatedTotalOperations(); } } else { // if we have a commit point, check that we have all the files listed in it in the blob store if (!commitPoints.commits().isEmpty()) { CommitPoint commitPoint = commitPoints.commits().get(0); boolean allTranslogFilesExists = true; for (CommitPoint.FileInfo fileInfo : commitPoint.translogFiles()) { if (!commitPointFileExistsInBlobs(fileInfo, blobs)) { allTranslogFilesExists = false; break; } } // if everything exists, we can seek forward in case there are new operations, otherwise, we copy over all again... if (allTranslogFilesExists) { translogCommitPointFiles.addAll(commitPoint.translogFiles()); if (snapshot.sameTranslogNewOperations()) { translogSnapshot.seekForward(snapshot.lastTranslogLength()); if (translogSnapshot.lengthInBytes() > 0) { snapshotRequired = true; expectedNumberOfOperations = translogSnapshot.estimatedTotalOperations() - snapshot.lastTotalTranslogOperations(); } } // else (no operations, nothing to snapshot) } else { // a full translog snapshot is required if (translogSnapshot.lengthInBytes() > 0) { expectedNumberOfOperations = translogSnapshot.estimatedTotalOperations(); snapshotRequired = true; } } } else { // no commit point, snapshot all the translog if (translogSnapshot.lengthInBytes() > 0) { expectedNumberOfOperations = translogSnapshot.estimatedTotalOperations(); snapshotRequired = true; } } } currentSnapshotStatus.translog().expectedNumberOfOperations(expectedNumberOfOperations); if (snapshotRequired) { CommitPoint.FileInfo addedTranslogFileInfo = new CommitPoint.FileInfo( fileNameFromGeneration(++generation), "translog-" + translogSnapshot.translogId(), translogSnapshot.lengthInBytes(), null /* no need for checksum in translog */); translogCommitPointFiles.add(addedTranslogFileInfo); try { snapshotTranslog(translogSnapshot, addedTranslogFileInfo); } catch (Exception e) { throw new IndexShardGatewaySnapshotFailedException(shardId, "Failed to snapshot translog", e); } } currentSnapshotStatus.translog() .time(System.currentTimeMillis() - currentSnapshotStatus.translog().startTime()); // now create and write the commit point currentSnapshotStatus.updateStage(SnapshotStatus.Stage.FINALIZE); long version = 0; if (!commitPoints.commits().isEmpty()) { version = commitPoints.commits().iterator().next().version() + 1; } String commitPointName = "commit-" + Long.toString(version, Character.MAX_RADIX); CommitPoint commitPoint = new CommitPoint(version, commitPointName, CommitPoint.Type.GENERATED, indexCommitPointFiles, translogCommitPointFiles); try { byte[] commitPointData = CommitPoints.toXContent(commitPoint); blobContainer.writeBlob(commitPointName, new FastByteArrayInputStream(commitPointData), commitPointData.length); } catch (Exception e) { throw new IndexShardGatewaySnapshotFailedException(shardId, "Failed to write commit point", e); } // delete all files that are not referenced by any commit point // build a new CommitPoint, that includes this one and all the saved ones List<CommitPoint> newCommitPointsList = Lists.newArrayList(); newCommitPointsList.add(commitPoint); for (CommitPoint point : commitPoints) { if (point.type() == CommitPoint.Type.SAVED) { newCommitPointsList.add(point); } } CommitPoints newCommitPoints = new CommitPoints(newCommitPointsList); // first, go over and delete all the commit points for (String blobName : blobs.keySet()) { if (!blobName.startsWith("commit-")) { continue; } long checkedVersion = Long.parseLong(blobName.substring("commit-".length()), Character.MAX_RADIX); if (!newCommitPoints.hasVersion(checkedVersion)) { try { blobContainer.deleteBlob(blobName); } catch (IOException e) { // ignore } } } // now go over all the blobs, and if they don't exists in a commit point, delete them for (String blobName : blobs.keySet()) { String name = blobName; if (!name.startsWith("__")) { continue; } if (blobName.contains(".part")) { name = blobName.substring(0, blobName.indexOf(".part")); } if (newCommitPoints.findNameFile(name) == null) { try { blobContainer.deleteBlob(blobName); } catch (IOException e) { // ignore, will delete it laters } } } }