Java tutorial
/* * Copyright 2013 Twitter, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.twitter.aurora.scheduler.configuration; import java.util.regex.Pattern; import javax.annotation.Nullable; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.apache.commons.lang.StringUtils; import com.twitter.aurora.gen.Constraint; import com.twitter.aurora.gen.JobConfiguration; import com.twitter.aurora.gen.LimitConstraint; import com.twitter.aurora.gen.TaskConfig; import com.twitter.aurora.gen.TaskConfig._Fields; import com.twitter.aurora.gen.TaskConstraint; import com.twitter.aurora.scheduler.base.JobKeys; import com.twitter.aurora.scheduler.storage.entities.IConstraint; import com.twitter.aurora.scheduler.storage.entities.IIdentity; import com.twitter.aurora.scheduler.storage.entities.IJobConfiguration; import com.twitter.aurora.scheduler.storage.entities.ITaskConfig; import com.twitter.aurora.scheduler.storage.entities.ITaskConstraint; import com.twitter.aurora.scheduler.storage.entities.IValueConstraint; import com.twitter.common.base.Closure; import com.twitter.common.base.MorePreconditions; import static com.twitter.aurora.gen.apiConstants.DEFAULT_ENVIRONMENT; import static com.twitter.aurora.gen.apiConstants.GOOD_IDENTIFIER_PATTERN_JVM; /** * Manages translation from a string-mapped configuration to a concrete configuration type, and * defaults for optional values. * * TODO(William Farner): Add input validation to all fields (strings not empty, positive ints, etc). */ public final class ConfigurationManager { public static final String DEDICATED_ATTRIBUTE = "dedicated"; @VisibleForTesting public static final String HOST_CONSTRAINT = "host"; @VisibleForTesting public static final String RACK_CONSTRAINT = "rack"; private static final Pattern GOOD_IDENTIFIER = Pattern.compile(GOOD_IDENTIFIER_PATTERN_JVM); private static final int MAX_IDENTIFIER_LENGTH = 255; private static class DefaultField implements Closure<TaskConfig> { private final _Fields field; private final Object defaultValue; DefaultField(_Fields field, Object defaultValue) { this.field = field; this.defaultValue = defaultValue; } @Override public void execute(TaskConfig task) { if (!task.isSet(field)) { task.setFieldValue(field, defaultValue); } } } private interface Validator<T> { void validate(T value) throws TaskDescriptionException; } private static class GreaterThan implements Validator<Number> { private final double min; private final String label; GreaterThan(double min, String label) { this.min = min; this.label = label; } @Override public void validate(Number value) throws TaskDescriptionException { if (this.min >= value.doubleValue()) { throw new TaskDescriptionException(label + " must be greater than " + this.min); } } } private static class RequiredFieldValidator<T> implements Validator<TaskConfig> { private final _Fields field; private final Validator<T> validator; RequiredFieldValidator(_Fields field, Validator<T> validator) { this.field = field; this.validator = validator; } public void validate(TaskConfig task) throws TaskDescriptionException { if (!task.isSet(field)) { throw new TaskDescriptionException("Field " + field.getFieldName() + " is required."); } @SuppressWarnings("unchecked") T value = (T) task.getFieldValue(field); validator.validate(value); } } private static final Iterable<Closure<TaskConfig>> DEFAULT_FIELD_POPULATORS = ImmutableList.of( new DefaultField(_Fields.IS_SERVICE, false), new DefaultField(_Fields.PRIORITY, 0), new DefaultField(_Fields.PRODUCTION, false), new DefaultField(_Fields.MAX_TASK_FAILURES, 1), new DefaultField(_Fields.TASK_LINKS, Maps.<String, String>newHashMap()), new DefaultField(_Fields.REQUESTED_PORTS, Sets.<String>newHashSet()), new DefaultField(_Fields.CONSTRAINTS, Sets.<Constraint>newHashSet()), new DefaultField(_Fields.ENVIRONMENT, DEFAULT_ENVIRONMENT), new Closure<TaskConfig>() { @Override public void execute(TaskConfig task) { if (!Iterables.any(task.getConstraints(), hasName(HOST_CONSTRAINT))) { task.addToConstraints(hostLimitConstraint(1)); } } }, new Closure<TaskConfig>() { @Override public void execute(TaskConfig task) { if (!isDedicated(ITaskConfig.build(task)) && task.isProduction() && task.isIsService() && !Iterables.any(task.getConstraints(), hasName(RACK_CONSTRAINT))) { task.addToConstraints(rackLimitConstraint(1)); } } }); private static final Iterable<RequiredFieldValidator<?>> REQUIRED_FIELDS_VALIDATORS = ImmutableList .<RequiredFieldValidator<?>>of( new RequiredFieldValidator<>(_Fields.NUM_CPUS, new GreaterThan(0.0, "num_cpus")), new RequiredFieldValidator<>(_Fields.RAM_MB, new GreaterThan(0.0, "ram_mb")), new RequiredFieldValidator<>(_Fields.DISK_MB, new GreaterThan(0.0, "disk_mb"))); private ConfigurationManager() { // Utility class. } @VisibleForTesting static boolean isGoodIdentifier(String identifier) { return GOOD_IDENTIFIER.matcher(identifier).matches() && (identifier.length() <= MAX_IDENTIFIER_LENGTH); } private static void checkNotNull(Object value, String error) throws TaskDescriptionException { if (value == null) { throw new TaskDescriptionException(error); } } private static void assertOwnerValidity(IIdentity jobOwner) throws TaskDescriptionException { checkNotNull(jobOwner, "No job owner specified!"); checkNotNull(jobOwner.getRole(), "No job role specified!"); checkNotNull(jobOwner.getUser(), "No job user specified!"); if (!isGoodIdentifier(jobOwner.getRole())) { throw new TaskDescriptionException("Job role contains illegal characters: " + jobOwner.getRole()); } if (!isGoodIdentifier(jobOwner.getUser())) { throw new TaskDescriptionException("Job user contains illegal characters: " + jobOwner.getUser()); } } private static String getRole(IValueConstraint constraint) { return Iterables.getOnlyElement(constraint.getValues()).split("/")[0]; } private static boolean isValueConstraint(ITaskConstraint taskConstraint) { return taskConstraint.getSetField() == TaskConstraint._Fields.VALUE; } public static boolean isDedicated(ITaskConfig task) { return Iterables.any(task.getConstraints(), getConstraintByName(DEDICATED_ATTRIBUTE)); } @Nullable private static IConstraint getDedicatedConstraint(ITaskConfig task) { return Iterables.find(task.getConstraints(), getConstraintByName(DEDICATED_ATTRIBUTE), null); } /** * Check validity of and populates defaults in a job configuration. This will return a deep copy * of the provided job configuration with default configuration values applied, and configuration * map values sanitized and applied to their respective struct fields. * * @param job Job to validate and populate. * @return A deep copy of {@code job} that has been populated. * @throws TaskDescriptionException If the job configuration is invalid. */ public static IJobConfiguration validateAndPopulate(IJobConfiguration job) throws TaskDescriptionException { Preconditions.checkNotNull(job); if (!job.isSetTaskConfig()) { throw new TaskDescriptionException("Job configuration must have taskConfig set."); } if (!job.isSetInstanceCount()) { throw new TaskDescriptionException("Job configuration does not have shardCount set."); } if (job.getInstanceCount() <= 0) { throw new TaskDescriptionException("Shard count must be positive."); } JobConfiguration builder = job.newBuilder(); assertOwnerValidity(job.getOwner()); if (!JobKeys.isValid(job.getKey())) { throw new TaskDescriptionException("Job key " + job.getKey() + " is invalid."); } if (!job.getKey().getRole().equals(job.getOwner().getRole())) { throw new TaskDescriptionException("Role in job key must match job owner."); } if (!isGoodIdentifier(job.getKey().getRole())) { throw new TaskDescriptionException("Job role contains illegal characters: " + job.getKey().getRole()); } if (!isGoodIdentifier(job.getKey().getEnvironment())) { throw new TaskDescriptionException( "Job environment contains illegal characters: " + job.getKey().getEnvironment()); } if (!isGoodIdentifier(job.getKey().getName())) { throw new TaskDescriptionException("Job name contains illegal characters: " + job.getKey().getName()); } builder.setTaskConfig(validateAndPopulate(ITaskConfig.build(builder.getTaskConfig())).newBuilder()); // Only one of [service=true, cron_schedule] may be set. if (!StringUtils.isEmpty(job.getCronSchedule()) && builder.getTaskConfig().isIsService()) { throw new TaskDescriptionException("A service task may not be run on a cron schedule: " + builder); } return IJobConfiguration.build(builder); } /** * Check validity of and populates defaults in a task configuration. This will return a deep copy * of the provided task configuration with default configuration values applied, and configuration * map values sanitized and applied to their respective struct fields. * * * @param config Task config to validate and populate. * @return A reference to the modified {@code config} (for chaining). * @throws TaskDescriptionException If the task is invalid. */ public static ITaskConfig validateAndPopulate(ITaskConfig config) throws TaskDescriptionException { TaskConfig builder = config.newBuilder(); if (!builder.isSetRequestedPorts()) { builder.setRequestedPorts(ImmutableSet.<String>of()); } maybeFillLinks(builder); assertOwnerValidity(config.getOwner()); if (!isGoodIdentifier(config.getJobName())) { throw new TaskDescriptionException("Job name contains illegal characters: " + config.getJobName()); } if (!isGoodIdentifier(config.getEnvironment())) { throw new TaskDescriptionException( "Environment contains illegal characters: " + config.getEnvironment()); } if (!builder.isSetExecutorConfig()) { throw new TaskDescriptionException("Configuration may not be null"); } // Maximize the usefulness of any thrown error message by checking required fields first. for (RequiredFieldValidator<?> validator : REQUIRED_FIELDS_VALIDATORS) { validator.validate(builder); } IConstraint constraint = getDedicatedConstraint(config); if (constraint != null) { if (!isValueConstraint(constraint.getConstraint())) { throw new TaskDescriptionException("A dedicated constraint must be of value type."); } IValueConstraint valueConstraint = constraint.getConstraint().getValue(); if (!(valueConstraint.getValues().size() == 1)) { throw new TaskDescriptionException("A dedicated constraint must have exactly one value"); } String dedicatedRole = getRole(valueConstraint); if (!config.getOwner().getRole().equals(dedicatedRole)) { throw new TaskDescriptionException( "Only " + dedicatedRole + " may use hosts dedicated for that role."); } } return ITaskConfig.build(applyDefaultsIfUnset(builder)); } /** * Provides a filter for the given constraint name. * * @param name The name of the constraint. * @return A filter that matches the constraint. */ public static Predicate<IConstraint> getConstraintByName(final String name) { return new Predicate<IConstraint>() { @Override public boolean apply(IConstraint constraint) { return constraint.getName().equals(name); } }; } @VisibleForTesting public static Constraint hostLimitConstraint(int limit) { return new Constraint(HOST_CONSTRAINT, TaskConstraint.limit(new LimitConstraint(limit))); } @VisibleForTesting public static Constraint rackLimitConstraint(int limit) { return new Constraint(RACK_CONSTRAINT, TaskConstraint.limit(new LimitConstraint(limit))); } private static Predicate<Constraint> hasName(final String name) { MorePreconditions.checkNotBlank(name); return new Predicate<Constraint>() { @Override public boolean apply(Constraint constraint) { return name.equals(constraint.getName()); } }; } /** * Applies defaults to unset values in a task. * * @param task Task to apply defaults to. * @return A reference to the (modified) {@code task}. */ @VisibleForTesting public static TaskConfig applyDefaultsIfUnset(TaskConfig task) { for (Closure<TaskConfig> populator : DEFAULT_FIELD_POPULATORS) { populator.execute(task); } return task; } /** * Applies defaults to unset values in a job and its tasks. * * @param job Job to apply defaults to. */ @VisibleForTesting public static void applyDefaultsIfUnset(JobConfiguration job) { ConfigurationManager.applyDefaultsIfUnset(job.getTaskConfig()); } private static void maybeFillLinks(TaskConfig task) { if (task.getTaskLinksSize() == 0) { ImmutableMap.Builder<String, String> links = ImmutableMap.builder(); if (task.getRequestedPorts().contains("health")) { links.put("health", "http://%host%:%port:health%"); } if (task.getRequestedPorts().contains("http")) { links.put("http", "http://%host%:%port:http%"); } task.setTaskLinks(links.build()); } } /** * Thrown when an invalid task or job configuration is encountered. */ public static class TaskDescriptionException extends Exception { public TaskDescriptionException(String msg) { super(msg); } } }