Java tutorial
/** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.aurora.scheduler.configuration.executor; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.List; import java.util.Optional; import java.util.stream.Stream; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.inject.AbstractModule; import org.apache.aurora.GuavaUtils; import org.apache.aurora.common.args.Arg; import org.apache.aurora.common.args.CmdLine; import org.apache.aurora.common.args.constraints.CanRead; import org.apache.aurora.common.args.constraints.Exists; import org.apache.aurora.common.base.MorePreconditions; import org.apache.aurora.common.quantity.Amount; import org.apache.aurora.common.quantity.Data; import org.apache.aurora.gen.Volume; import org.apache.aurora.gen.apiConstants; import org.apache.aurora.scheduler.resources.ResourceType; import org.apache.mesos.v1.Protos; import org.apache.mesos.v1.Protos.CommandInfo; import org.apache.mesos.v1.Protos.CommandInfo.URI; import org.apache.mesos.v1.Protos.ExecutorInfo; import org.apache.mesos.v1.Protos.Resource; import org.apache.mesos.v1.Protos.Value.Scalar; import org.apache.mesos.v1.Protos.Value.Type; import static org.apache.aurora.scheduler.resources.ResourceType.CPUS; import static org.apache.aurora.scheduler.resources.ResourceType.RAM_MB; /** * Binding module for {@link ExecutorSettings}. */ public class ExecutorModule extends AbstractModule { @CmdLine(name = "custom_executor_config", help = "Path to custom executor settings configuration file.") @Exists @CanRead private static final Arg<File> CUSTOM_EXECUTOR_CONFIG = Arg.create(null); @CmdLine(name = "thermos_executor_path", help = "Path to the thermos executor entry point.") private static final Arg<String> THERMOS_EXECUTOR_PATH = Arg.create(); @CmdLine(name = "thermos_executor_resources", help = "A comma separated list of additional resources to copy into the sandbox." + "Note: if thermos_executor_path is not the thermos_executor.pex file itself, " + "this must include it.") private static final Arg<List<String>> THERMOS_EXECUTOR_RESOURCES = Arg.create(ImmutableList.of()); @CmdLine(name = "thermos_executor_flags", help = "Extra arguments to be passed to the thermos executor") private static final Arg<String> THERMOS_EXECUTOR_FLAGS = Arg.create(null); @CmdLine(name = "thermos_home_in_sandbox", help = "If true, changes HOME to the sandbox before running the executor. " + "This primarily has the effect of causing the executor and runner " + "to extract themselves into the sandbox.") private static final Arg<Boolean> THERMOS_HOME_IN_SANDBOX = Arg.create(false); /** * Extra CPU allocated for each executor. */ @CmdLine(name = "thermos_executor_cpu", help = "The number of CPU cores to allocate for each instance of the executor.") private static final Arg<Double> EXECUTOR_OVERHEAD_CPUS = Arg.create(0.25); /** * Extra RAM allocated for the executor. */ @CmdLine(name = "thermos_executor_ram", help = "The amount of RAM to allocate for each instance of the executor.") private static final Arg<Amount<Long, Data>> EXECUTOR_OVERHEAD_RAM = Arg.create(Amount.of(128L, Data.MB)); @CmdLine(name = "global_container_mounts", help = "A comma separated list of mount points (in host:container form) to mount " + "into all (non-mesos) containers.") private static final Arg<List<Volume>> GLOBAL_CONTAINER_MOUNTS = Arg.create(ImmutableList.of()); @CmdLine(name = "populate_discovery_info", help = "If true, Aurora populates DiscoveryInfo field of Mesos TaskInfo.") private static final Arg<Boolean> POPULATE_DISCOVERY_INFO = Arg.create(false); @VisibleForTesting static CommandInfo makeExecutorCommand(String thermosExecutorPath, List<String> thermosExecutorResources, boolean thermosHomeInSandbox, String thermosExecutorFlags) { Stream<String> resourcesToFetch = Stream.concat(ImmutableList.of(thermosExecutorPath).stream(), thermosExecutorResources.stream()); StringBuilder sb = new StringBuilder(); if (thermosHomeInSandbox) { sb.append("HOME=${MESOS_SANDBOX=.} "); } // Default to the value of $MESOS_SANDBOX if present. This is necessary for docker tasks, // in which case the mesos agent is responsible for setting $MESOS_SANDBOX. sb.append("${MESOS_SANDBOX=.}/"); sb.append(uriBasename(thermosExecutorPath)); sb.append(" "); sb.append(Optional.ofNullable(thermosExecutorFlags).orElse("")); return CommandInfo.newBuilder().setValue(sb.toString().trim()) .addAllUris(resourcesToFetch.map(r -> URI.newBuilder().setValue(r).setExecutable(true).build()) .collect(GuavaUtils.toImmutableList())) .build(); } private static ExecutorConfig makeThermosExecutorConfig() { List<Protos.Volume> volumeMounts = ImmutableList.<Protos.Volume>builder() .addAll(Iterables.transform(GLOBAL_CONTAINER_MOUNTS.get(), v -> Protos.Volume.newBuilder().setHostPath(v.getHostPath()) .setContainerPath(v.getContainerPath()) .setMode(Protos.Volume.Mode.valueOf(v.getMode().getValue())).build())) .build(); return new ExecutorConfig(ExecutorInfo.newBuilder().setName(apiConstants.AURORA_EXECUTOR_NAME) // Necessary as executorId is a required field. .setExecutorId(Executors.PLACEHOLDER_EXECUTOR_ID) .setCommand(makeExecutorCommand(THERMOS_EXECUTOR_PATH.get(), THERMOS_EXECUTOR_RESOURCES.get(), THERMOS_HOME_IN_SANDBOX.get(), THERMOS_EXECUTOR_FLAGS.get())) .addResources(makeResource(CPUS, EXECUTOR_OVERHEAD_CPUS.get())) .addResources(makeResource(RAM_MB, EXECUTOR_OVERHEAD_RAM.get().as(Data.MB))).build(), volumeMounts, "thermos-"); } private static ExecutorSettings makeExecutorSettings() { try { ImmutableMap.Builder<String, ExecutorConfig> configsBuilder = ImmutableMap.builder(); configsBuilder.put(apiConstants.AURORA_EXECUTOR_NAME, makeThermosExecutorConfig()); if (CUSTOM_EXECUTOR_CONFIG.hasAppliedValue()) { configsBuilder.putAll(ExecutorSettingsLoader.read( Files.newBufferedReader(CUSTOM_EXECUTOR_CONFIG.get().toPath(), StandardCharsets.UTF_8))); } return new ExecutorSettings(configsBuilder.build(), POPULATE_DISCOVERY_INFO.get()); } catch (ExecutorSettingsLoader.ExecutorConfigException | IOException e) { throw new IllegalArgumentException("Failed to read executor settings: " + e, e); } } @Override protected void configure() { bind(ExecutorSettings.class).toInstance(makeExecutorSettings()); } private static Resource makeResource(ResourceType type, double value) { return Resource.newBuilder().setType(Type.SCALAR).setName(type.getMesosName()) .setScalar(Scalar.newBuilder().setValue(value)).build(); } private static String uriBasename(String uri) { int lastSlash = uri.lastIndexOf('/'); if (lastSlash == -1) { return uri; } else { String basename = uri.substring(lastSlash + 1); MorePreconditions.checkNotBlank(basename, "URI must not end with a slash."); return basename; } } }