com.cloudera.director.aws.ec2.EphemeralDeviceMappings.java Source code

Java tutorial

Introduction

Here is the source code for com.cloudera.director.aws.ec2.EphemeralDeviceMappings.java

Source

// (c) Copyright 2015 Cloudera, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.cloudera.director.aws.ec2;

import static com.cloudera.director.aws.ec2.EphemeralDeviceMappings.EphemeralDeviceMappingsConfigProperties.EphemeralDeviceMappingsConfigurationPropertyToken.CUSTOM_MAPPINGS_PATH;
import static com.cloudera.director.aws.ec2.EphemeralDeviceMappings.EphemeralDeviceMappingsConfigProperties.EphemeralDeviceMappingsConfigurationPropertyToken.DEVICE_NAME_PREFIX;
import static com.cloudera.director.aws.ec2.EphemeralDeviceMappings.EphemeralDeviceMappingsConfigProperties.EphemeralDeviceMappingsConfigurationPropertyToken.RANGE_START;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;

import com.amazonaws.services.ec2.model.BlockDeviceMapping;
import com.cloudera.director.aws.common.PropertyResolvers;
import com.cloudera.director.spi.v1.model.ConfigurationProperty;
import com.cloudera.director.spi.v1.model.Configured;
import com.cloudera.director.spi.v1.model.LocalizationContext;
import com.cloudera.director.spi.v1.model.util.ChildLocalizationContext;
import com.cloudera.director.spi.v1.model.util.SimpleConfiguration;
import com.cloudera.director.spi.v1.model.util.SimpleConfigurationPropertyBuilder;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Functions;
import com.google.common.base.Optional;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.io.Files;

import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;

import javax.annotation.Nonnull;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.env.PropertyResolver;

/**
 * Maps an instance type to a list of block device mappings for ephemeral
 * storage. EC2 does not provide a means through the API to discover mappings.
 * This class looks up mappings using a built-in list and an override list that
 * the user can easily modify.
 *
 * @see <a href="http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/InstanceStorage.html">Amazon EC2 Instance Stor</a>
 * @see <a href="http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html">Block Device Mapping</a>
 */
@SuppressWarnings("PMD.TooManyStaticImports")
public class EphemeralDeviceMappings implements Function<String, List<BlockDeviceMapping>> {

    private static final Logger LOG = LoggerFactory.getLogger(EphemeralDeviceMappings.class);

    public static class EphemeralDeviceMappingsConfigProperties {

        /**
         * Ephemeral device mappings configuration properties.
         */
        // Fully qualifying class name due to compiler bug
        public static enum EphemeralDeviceMappingsConfigurationPropertyToken
                implements com.cloudera.director.spi.v1.model.ConfigurationPropertyToken {

            /**
             * The device name prefix for ephemeral drives.
             */
            DEVICE_NAME_PREFIX(new SimpleConfigurationPropertyBuilder().configKey("deviceNamePrefix")
                    .name("Device name prefix").defaultDescription("The device name prefix for ephemeral drives.")
                    .build()),

            /**
             * The range start for attaching ephemeral drives.
             */
            RANGE_START(new SimpleConfigurationPropertyBuilder().configKey("rangeStart").name("Range start")
                    .defaultDescription("The range start for attaching ephemeral drives.").build()),

            /**
             * Path for the custom device mappings file. Relative paths are based on
             * the plugin configuration directory.
             */
            CUSTOM_MAPPINGS_PATH(new SimpleConfigurationPropertyBuilder().configKey("customMappingsPath")
                    .name("Custom mappings path")
                    .defaultDescription("The path for the custom ephemeral device mappings file. Relative "
                            + "paths are based on the plugin configuration directory.")
                    .build());

            /**
             * The configuration property.
             */
            private final ConfigurationProperty configurationProperty;

            /**
             * Creates a configuration property token with the specified parameters.
             *
             * @param configurationProperty the configuration property
             */
            private EphemeralDeviceMappingsConfigurationPropertyToken(ConfigurationProperty configurationProperty) {
                this.configurationProperty = configurationProperty;
            }

            @Override
            public ConfigurationProperty unwrap() {
                return configurationProperty;
            }
        }

        public static final String DEFAULT_DEVICE_NAME_PREFIX = "/dev/sd";

        /**
         * Ephemeral drives should be attached in the following range:
         * /dev/(sd|xvd)[b-y].
         */
        public static final char DEFAULT_RANGE_START_FOR_EPHEMERAL_DRIVES = 'b';

        public static final String DEFAULT_CUSTOM_MAPPINGS_PATH = "ec2.ephemeraldevicemappings.properties";

        private final File configurationDirectory;

        private String deviceNamePrefix = DEFAULT_DEVICE_NAME_PREFIX;
        private char rangeStart = DEFAULT_RANGE_START_FOR_EPHEMERAL_DRIVES;
        private String customMappingsPath = DEFAULT_CUSTOM_MAPPINGS_PATH;

        /**
         * Creates ephemeral device mappings config properties with the specified parameters.
         *
         * @param configuration            the configuration
         * @param configurationDirectory   the plugin configuration directory
         * @param cloudLocalizationContext the parent cloud localization context
         */
        public EphemeralDeviceMappingsConfigProperties(Configured configuration, File configurationDirectory,
                LocalizationContext cloudLocalizationContext) {
            this.configurationDirectory = configurationDirectory;
            LocalizationContext localizationContext = new ChildLocalizationContext(cloudLocalizationContext,
                    "ephemeralDeviceMappings");
            setDeviceNamePrefix(configuration.getConfigurationValue(DEVICE_NAME_PREFIX, localizationContext));
            setRangeStart(configuration.getConfigurationValue(RANGE_START, localizationContext));
            setCustomMappingsPath(configuration.getConfigurationValue(CUSTOM_MAPPINGS_PATH, localizationContext));
        }

        public String getDeviceNamePrefix() {
            return deviceNamePrefix;
        }

        public void setDeviceNamePrefix(String deviceNamePrefix) {
            if (deviceNamePrefix != null) {
                LOG.info("Overriding deviceNamePrefix={} (default {})", deviceNamePrefix,
                        DEFAULT_DEVICE_NAME_PREFIX);
                this.deviceNamePrefix = deviceNamePrefix;
            }
        }

        public char getRangeStart() {
            return rangeStart;
        }

        public void setRangeStart(String rangeStart) {
            if (rangeStart != null) {
                if (rangeStart.length() != 1) {
                    throw new IllegalArgumentException("rangeStart must be a single character");
                }
                char c = rangeStart.charAt(0);
                LOG.info("Overriding rangeStart={} (default {})", c, DEFAULT_RANGE_START_FOR_EPHEMERAL_DRIVES);
                this.rangeStart = c;
            }
        }

        public String getCustomMappingsPath() {
            return customMappingsPath;
        }

        File getCustomMappingsFile() {
            return getCustomMappingsFile(customMappingsPath);
        }

        File getCustomMappingsFile(String customMappingsPath) {
            File customMappingsPathFile = new File(customMappingsPath);
            return customMappingsPathFile.isAbsolute() ? customMappingsPathFile
                    : new File(configurationDirectory, customMappingsPath);
        }

        public void setCustomMappingsPath(String customMappingsPath) {
            if (customMappingsPath != null) {
                LOG.info("Overriding customMappingsPath={} (default {})", customMappingsPath,
                        DEFAULT_CUSTOM_MAPPINGS_PATH);
                checkArgument(getCustomMappingsFile(customMappingsPath).exists(),
                        "Custom ephemeral device mappings path " + customMappingsPath + " does not exist");
                this.customMappingsPath = customMappingsPath;
            }
        }
    }

    public static class EphemeralDeviceMappingsConfig {

        public static final String BUILT_IN_LOCATION = "classpath:/com/cloudera/director/aws/ec2/ephemeraldevicemappings.properties";

        protected EphemeralDeviceMappingsConfigProperties ephemeralDeviceMappingsConfigProperties;

        public PropertyResolver ephemeralDeviceMappingsResolver() {
            try {
                return PropertyResolvers.newMultiResourcePropertyResolver(BUILT_IN_LOCATION, "file:"
                        + ephemeralDeviceMappingsConfigProperties.getCustomMappingsFile().getAbsolutePath());
            } catch (IOException e) {
                throw new IllegalArgumentException("Could not load ephemeral device mappings", e);
            }
        }

        /**
         * Creates an ephemeral device mappings config with the specified parameters.
         *
         * @param ephemeralDeviceMappingsConfigProperties the config properties
         */
        public EphemeralDeviceMappingsConfig(
                EphemeralDeviceMappingsConfigProperties ephemeralDeviceMappingsConfigProperties) {
            this.ephemeralDeviceMappingsConfigProperties = ephemeralDeviceMappingsConfigProperties;
        }
    }

    EphemeralDeviceMappingsConfigProperties ephemeralDeviceMappingsConfigProperties;

    PropertyResolver ephemeralDeviceMappingsResolver;

    /**
     * Creates ephemeral device mappings with the specified parameters.
     *
     * @param configuration               the configuration
     * @param configurationDirectory      the plugin configuration directory
     * @param launcherLocalizationContext the parent launcher localization context
     */
    public EphemeralDeviceMappings(Configured configuration, File configurationDirectory,
            LocalizationContext launcherLocalizationContext) {
        this(new EphemeralDeviceMappingsConfigProperties(configuration, configurationDirectory,
                launcherLocalizationContext));
    }

    /**
     * Creates ephemeral device mappings with the specified parameters.
     *
     * @param ephemeralDeviceMappingsConfigProperties the config properties
     */
    public EphemeralDeviceMappings(
            EphemeralDeviceMappingsConfigProperties ephemeralDeviceMappingsConfigProperties) {
        this(ephemeralDeviceMappingsConfigProperties,
                new EphemeralDeviceMappingsConfig(ephemeralDeviceMappingsConfigProperties)
                        .ephemeralDeviceMappingsResolver());
    }

    /**
     * Creates ephemeral device mappings with the specified parameters.
     *
     * @param ephemeralDeviceMappingsConfigProperties the config properties
     * @param ephemeralDeviceMappingsResolver         the ephemeral device mappings resolver
     */
    private EphemeralDeviceMappings(EphemeralDeviceMappingsConfigProperties ephemeralDeviceMappingsConfigProperties,
            PropertyResolver ephemeralDeviceMappingsResolver) {
        this.ephemeralDeviceMappingsConfigProperties = ephemeralDeviceMappingsConfigProperties;
        this.ephemeralDeviceMappingsResolver = ephemeralDeviceMappingsResolver;
    }

    /**
     * Generates a list of block device mappings for all ephemeral drives for
     * the given instance type.
     *
     * @param instanceType EC2 instance type
     * @return list of block device mappings
     * @see <a href="http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html">Block Device Mapping</a>
     */
    @Nonnull
    public List<BlockDeviceMapping> apply(String instanceType) {
        checkNotNull(instanceType, "instanceType is null");

        Optional<Integer> optCount = getCount(instanceType);
        if (!optCount.isPresent()) {
            LOG.error("Unsupported instance type {}, add its ephemeral instance "
                    + "volume count as a custom mapping; assuming zero", instanceType);
            return Collections.emptyList();
        }

        int count = optCount.get();
        if (count == 0) {
            return Collections.emptyList();
        }

        List<String> deviceNames = getLinuxDeviceNames(
                ephemeralDeviceMappingsConfigProperties.getDeviceNamePrefix(),
                ephemeralDeviceMappingsConfigProperties.getRangeStart(), count);

        List<BlockDeviceMapping> result = Lists.newArrayListWithExpectedSize(count);
        int index = 0;
        for (String device : deviceNames) {
            result.add(new BlockDeviceMapping().withDeviceName(device).withVirtualName("ephemeral" + index));
            index += 1;
        }

        return result;
    }

    private Optional<Integer> getCount(String instanceType) {
        return Optional.fromNullable(ephemeralDeviceMappingsResolver.getProperty(instanceType, Integer.class));
    }

    @VisibleForTesting
    List<String> getLinuxDeviceNames(String instanceType) {
        Optional<Integer> count = getCount(instanceType);
        if (!count.isPresent() || count.get() == 0) {
            return Collections.emptyList();
        }

        return getLinuxDeviceNames(EphemeralDeviceMappingsConfigProperties.DEFAULT_DEVICE_NAME_PREFIX,
                EphemeralDeviceMappingsConfigProperties.DEFAULT_RANGE_START_FOR_EPHEMERAL_DRIVES, count.get());
    }

    @VisibleForTesting
    List<String> getLinuxDeviceNames(String pathPrefix, char startSuffix, int count) {
        List<String> result = Lists.newArrayListWithExpectedSize(count);
        for (int i = 0; i < count; i++) {
            char suffix = (char) (startSuffix + i);
            result.add(pathPrefix + suffix);
        }
        return result;
    }

    /**
     * Gets a test instance of this class that uses only the given mapping.
     *
     * @param counts                      map of instance types to counts
     * @param launcherLocalizationContext the parent launcher localization context
     * @return new mapping object
     */
    public static EphemeralDeviceMappings getTestInstance(Map<String, Integer> counts,
            LocalizationContext launcherLocalizationContext) {
        Map<String, String> propertyMap = Maps.transformValues(counts, Functions.toStringFunction());
        PropertyResolver ephemeralDeviceMappingsResolver = PropertyResolvers.newMapPropertyResolver(propertyMap);
        File tempDir = Files.createTempDir();
        tempDir.deleteOnExit();
        EphemeralDeviceMappingsConfigProperties ephemeralDeviceMappingsConfigProperties = new EphemeralDeviceMappingsConfigProperties(
                new SimpleConfiguration(), tempDir, launcherLocalizationContext);
        return new EphemeralDeviceMappings(ephemeralDeviceMappingsConfigProperties,
                ephemeralDeviceMappingsResolver);
    }
}