Java tutorial
/** * Copyright (C) 2009-2015 Dell, Inc. * See annotations for authorship information * * ==================================================================== * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * ==================================================================== */ package org.dasein.cloud.aws; import org.apache.commons.codec.binary.Base64; import org.apache.commons.codec.binary.Hex; import org.apache.http.*; import org.apache.http.client.HttpClient; import org.apache.http.client.entity.GzipDecompressingEntity; import org.apache.http.client.utils.URIUtils; import org.apache.http.client.utils.URLEncodedUtils; import org.apache.http.conn.params.ConnRoutePNames; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.params.BasicHttpParams; import org.apache.http.params.HttpParams; import org.apache.http.params.HttpProtocolParams; import org.apache.http.protocol.HttpContext; import org.apache.log4j.Logger; import org.dasein.cloud.*; import org.dasein.cloud.aws.admin.AWSAdminServices; import org.dasein.cloud.aws.compute.EC2ComputeServices; import org.dasein.cloud.aws.compute.EC2Exception; import org.dasein.cloud.aws.compute.EC2Method; import org.dasein.cloud.aws.identity.AWSIdentityServices; import org.dasein.cloud.aws.identity.IAMMethod; import org.dasein.cloud.aws.network.EC2NetworkServices; import org.dasein.cloud.aws.network.ELBMethod; import org.dasein.cloud.aws.platform.AWSPlatformServices; import org.dasein.cloud.aws.storage.AWSCloudStorageServices; import org.dasein.cloud.aws.storage.S3Method; import org.dasein.cloud.compute.ComputeServices; import org.dasein.cloud.compute.VirtualMachineSupport; import org.dasein.cloud.platform.KeyValuePair; import org.dasein.cloud.storage.BlobStoreSupport; import org.dasein.cloud.storage.StorageServices; import org.dasein.cloud.util.APITrace; import org.w3c.dom.Document; import org.w3c.dom.Node; import org.w3c.dom.NodeList; import javax.annotation.Nonnull; import javax.annotation.Nullable; import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URI; import java.net.URISyntaxException; import java.net.URLEncoder; import java.security.InvalidKeyException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.*; public class AWSCloud extends AbstractCloud { private static final int MAX_RETRIES = 0; static private String getLastItem(String name) { int idx = name.lastIndexOf('.'); if (idx < 0) { return name; } else if (idx == (name.length() - 1)) { return ""; } return name.substring(idx + 1); } static public Logger getLogger(Class<?> cls) { String pkg = getLastItem(cls.getPackage().getName()); if (pkg.equals("aws")) { pkg = ""; } else { pkg = pkg + "."; } return Logger.getLogger("dasein.cloud.aws.std." + pkg + getLastItem(cls.getName())); } static public Logger getWireLogger(Class<?> cls) { return Logger.getLogger("dasein.cloud.aws.wire." + getLastItem(cls.getPackage().getName()) + "." + getLastItem(cls.getName())); } static private final Logger logger = getLogger(AWSCloud.class); static public final String P_ACCESS = "AWSAccessKeyId"; static public final String P_ACTION = "Action"; static public final String P_CFAUTH = "Authorization"; static public final String P_AWS_DATE = "x-amz-date"; static public final String P_AWS_CONTENT_SHA256 = "x-amz-content-sha256"; static public final String P_GOOG_DATE = "x-goog-date"; static public final String P_SIGNATURE = "Signature"; static public final String P_SIGNATURE_METHOD = "SignatureMethod"; static public final String P_SIGNATURE_VERSION = "SignatureVersion"; static public final String P_TIMESTAMP = "Timestamp"; static public final String P_VERSION = "Version"; static public final String CLOUD_FRONT_ALGORITHM = "HmacSHA1"; static public final String EC2_ALGORITHM = "HmacSHA256"; static public final String S3_ALGORITHM = "HmacSHA1"; static public final String SIGNATURE_V2 = "2"; static public final String SIGNATURE_V4 = "4"; static public final String V4_ALGORITHM = "AWS4-HMAC-SHA256"; static public final String V4_TERMINATION = "aws4_request"; static public final String PLATFORM_EC2 = "EC2"; static public final String PLATFORM_VPC = "VPC"; static public @Nonnull String encode(@Nonnull String value, boolean encodePath) throws InternalException { String encoded; try { encoded = URLEncoder.encode(value, "utf-8").replace("+", "%20").replace("*", "%2A").replace("%7E", "~"); if (encodePath) { encoded = encoded.replace("%2F", "/"); } } catch (UnsupportedEncodingException e) { logger.error(e); e.printStackTrace(); throw new InternalException(e); } return encoded; } static public String escapeXml(String nonxml) { StringBuilder str = new StringBuilder(); for (int i = 0; i < nonxml.length(); i++) { char c = nonxml.charAt(i); switch (c) { case '&': str.append("&"); break; case '>': str.append(">"); break; case '<': str.append("<"); break; case '"': str.append("""); break; case '[': str.append("["); break; case ']': str.append("]"); break; case '!': str.append("!"); break; default: str.append(c); } } return str.toString(); } static public byte[] HmacSHA256(String data, byte[] key) throws InternalException { final String algorithm = "HmacSHA256"; Mac mac; try { mac = Mac.getInstance(algorithm); mac.init(new SecretKeySpec(key, algorithm)); return mac.doFinal(data.getBytes("UTF-8")); } catch (NoSuchAlgorithmException e) { throw new InternalException(e); } catch (InvalidKeyException e) { throw new InternalException(e); } catch (UnsupportedEncodingException e) { throw new InternalException(e); } } static public String computeSHA256Hash(String value) throws InternalException { try { byte[] valueBytes = value.getBytes("utf-8"); BufferedInputStream inputStream = new BufferedInputStream(new ByteArrayInputStream(valueBytes)); MessageDigest digest = MessageDigest.getInstance("SHA-256"); byte[] buffer = new byte[4096]; int read; while ((read = inputStream.read(buffer, 0, buffer.length)) != -1) { digest.update(buffer, 0, read); } return new String(Hex.encodeHex(digest.digest(), true)); } catch (NoSuchAlgorithmException e) { throw new InternalException(e); } catch (IOException e) { throw new InternalException(e); } } public AWSCloud() { } private String buildEc2AuthString(String method, String serviceUrl, Map<String, String> parameters) throws InternalException { StringBuilder authString = new StringBuilder(); TreeSet<String> sortedKeys; URI endpoint; String tmp; authString.append(method); authString.append("\n"); try { endpoint = new URI(serviceUrl); } catch (URISyntaxException e) { logger.error(e); e.printStackTrace(); throw new InternalException(e); } authString.append(endpoint.getHost().toLowerCase()); authString.append("\n"); tmp = endpoint.getPath(); if (tmp == null || tmp.length() == 0) { tmp = "/"; } authString.append(encode(tmp, true)); authString.append("\n"); sortedKeys = new TreeSet<String>(); sortedKeys.addAll(parameters.keySet()); boolean first = true; for (String key : sortedKeys) { String value = parameters.get(key); if (!first) { authString.append("&"); } else { first = false; } authString.append(encode(key, false)); authString.append("="); if (value == null) { value = ""; } authString.append(encode(value, false)); } return authString.toString(); } public boolean createTags(String service, String resourceId, Tag... keyValuePairs) { return createTags(service, new String[] { resourceId }, keyValuePairs); } public boolean createTags(final String service, final String[] resourceIds, final Tag... keyValuePairs) { // TODO(stas): de-async experiment boolean async = false; if (async) { hold(); Thread t = new Thread() { public void run() { try { createTags(1, service, resourceIds, keyValuePairs); } finally { release(); } } }; t.setName("Tag Setter"); t.setDaemon(true); t.start(); } else { createTags(1, service, resourceIds, keyValuePairs); } return true; } private void createTags(int attempt, String service, String[] resourceIds, Tag... keyValuePairs) { APITrace.begin(this, "Cloud.createTags"); try { try { Map<String, String> parameters = null, tagParameters = null; if (service.equalsIgnoreCase(ELBMethod.SERVICE_ID)) { parameters = getElbParameters(getContext(), "AddTags"); addIndexedParameters(parameters, "LoadBalancerNames.member.", resourceIds); tagParameters = getTagsFromKeyValuePairs("Tags.member.", keyValuePairs); } else if (service.equalsIgnoreCase("rds")) { parameters = getStandardRdsParameters(getContext(), "AddTagsToResource"); // We can't tag multiple RDS resource at a time. parameters.put("ResourceName", resourceIds[0]); tagParameters = getTagsFromKeyValuePairs("Tags.member.", keyValuePairs); } else { parameters = getStandardParameters(getContext(), "CreateTags"); addIndexedParameters(parameters, "ResourceId.", resourceIds); tagParameters = getTagsFromKeyValuePairs("Tag.", keyValuePairs); } if (tagParameters.size() == 0) { return; } addExtraParameters(parameters, tagParameters); EC2Method method = new EC2Method(service, this, parameters); try { method.invoke(); } catch (EC2Exception e) { if (attempt > MAX_RETRIES) { logger.error("EC2 error setting tags for " + Arrays.toString(resourceIds) + ": " + e.getSummary()); return; } try { Thread.sleep(5000L); } catch (InterruptedException ignore) { } logger.warn("Retry attempt " + (attempt + 1) + " to create tags for [" + resourceIds + "]"); createTags(attempt + 1, service, resourceIds, keyValuePairs); } } catch (Throwable ignore) { logger.error("Error while creating tags for " + Arrays.toString(resourceIds) + ".", ignore); } } finally { APITrace.end(); } } private Map<String, String> getTagsFromKeyValuePairs(String tagPrefix, Tag... keyValuePairs) { Map<String, String> tagParameters = new HashMap<String, String>(); for (int i = 0; i < keyValuePairs.length; i++) { String key = keyValuePairs[i].getKey(); String value = keyValuePairs[i].getValue(); tagParameters.put(tagPrefix + (i + 1) + ".Key", key); tagParameters.put(tagPrefix + (i + 1) + ".Value", value != null ? value : ""); } return tagParameters; } public void createTagsSynchronously(final String resourceId, final Tag... keyValuePairs) throws CloudException, InternalException { createTagsSynchronously(new String[] { resourceId }, keyValuePairs); } public void createTagsSynchronously(final String[] resourceIds, final Tag... keyValuePairs) throws CloudException, InternalException { APITrace.begin(this, "Cloud.createTagsSynchronously"); try { Map<String, String> parameters = getStandardParameters(getContext(), "CreateTags"); addIndexedParameters(parameters, "ResourceId.", resourceIds); Map<String, String> tagParameters = getTagsFromKeyValuePairs("Tag.", keyValuePairs); if (tagParameters.size() == 0) { return; } addExtraParameters(parameters, tagParameters); new EC2Method(EC2Method.SERVICE_ID, this, parameters).invoke(); } finally { APITrace.end(); } } public boolean removeTags(String service, String resourceId, Tag... keyValuePairs) { return removeTags(service, new String[] { resourceId }, keyValuePairs); } public boolean removeTags(String service, String[] resourceIds, Tag... keyValuePairs) { APITrace.begin(this, "Cloud.removeTags"); try { try { Map<String, String> parameters, tagParameters = null; if (service.equalsIgnoreCase(ELBMethod.SERVICE_ID)) { parameters = getElbParameters(getContext(), "RemoveTags"); addIndexedParameters(parameters, "LoadBalancerNames.member.", resourceIds); tagParameters = getTagsFromKeyValuePairs("Tags.member.", keyValuePairs); } else if (service.equalsIgnoreCase("rds")) { parameters = getStandardRdsParameters(getContext(), "RemoveTagsFromResource"); parameters.put("ResourceName", resourceIds[0]); for (int i = 0; i < keyValuePairs.length; i++) parameters.put("TagKeys.member." + (i + 1), keyValuePairs[i].getKey()); } else { parameters = getStandardParameters(getContext(), "DeleteTags"); addIndexedParameters(parameters, "ResourceId.", resourceIds); tagParameters = getTagsFromKeyValuePairs("Tag.", keyValuePairs); } addExtraParameters(parameters, tagParameters); EC2Method method = new EC2Method(service, this, parameters); method.invoke(); return true; } catch (Throwable ignore) { logger.error("Error while removing tags for " + Arrays.toString(resourceIds) + ".", ignore); return false; } } finally { APITrace.end(); } } public Map<String, String> getTagsFromTagSet(Node attr) { if (attr == null || !attr.hasChildNodes()) { return null; } Map<String, String> tags = new HashMap<String, String>(); NodeList tagNodes = attr.getChildNodes(); for (int j = 0; j < tagNodes.getLength(); j++) { Tag t = toTag(tagNodes.item(j)); if (t != null) { tags.put(t.getKey(), t.getValue()); } } return tags; } public Tag toTag(@Nonnull Node tag) { if (tag.getNodeName().equals("item") && tag.hasChildNodes()) { NodeList parts = tag.getChildNodes(); String key = null, value = null; for (int k = 0; k < parts.getLength(); k++) { Node part = parts.item(k); if (part.getNodeName().equalsIgnoreCase("key")) { if (part.hasChildNodes()) { key = part.getFirstChild().getNodeValue().trim(); } } else if (part.getNodeName().equalsIgnoreCase("value")) { if (part.hasChildNodes()) { value = part.getFirstChild().getNodeValue().trim(); } } } if (key != null && value != null) { return new Tag(key, value); } } return null; } @Override public AWSAdminServices getAdminServices() { EC2Provider p = getEC2Provider(); if (p.isAWS() || p.isEnStratus() || p.isOpenStack() || p.isEucalyptus()) { return new AWSAdminServices(this); } return null; } private @Nonnull String[] getBootstrapUrls(@Nullable ProviderContext ctx) { String endpoint = (ctx == null ? null : ctx.getCloud().getEndpoint()); if (endpoint == null) { return new String[0]; } if (!endpoint.contains(",")) { return new String[] { endpoint }; } String[] endpoints = endpoint.split(","); if (endpoints == null) { endpoints = new String[0]; } if (endpoints.length > 1) { String second = endpoints[1]; if (!second.startsWith("http")) { if (endpoints[0].startsWith("http")) { // likely a URL with a , in it return new String[] { endpoint + (getEC2Provider().isEucalyptus() ? "/Eucalyptus" : "") }; } } } for (int i = 0; i < endpoints.length; i++) { if (!endpoints[i].startsWith("http")) { endpoints[i] = "https://" + endpoints[i] + (getEC2Provider().isEucalyptus() ? "/Eucalyptus" : ""); } } return endpoints; } @Override public @Nonnull String getCloudName() { ProviderContext ctx = getContext(); String name = (ctx == null ? null : ctx.getCloud().getCloudName()); return ((name == null) ? "AWS" : name); } @Override public EC2ComputeServices getComputeServices() { if (getEC2Provider().isStorage()) { return null; } return new EC2ComputeServices(this); } static public final String DSN_ACCESS_KEY = "accessKey"; @Override public @Nonnull ContextRequirements getContextRequirements() { return new ContextRequirements( new ContextRequirements.Field(DSN_ACCESS_KEY, "AWS API access keys", ContextRequirements.FieldType.KEYPAIR, ContextRequirements.Field.ACCESS_KEYS, true), new ContextRequirements.Field("proxyHost", "Proxy host", ContextRequirements.FieldType.TEXT, false), new ContextRequirements.Field("proxyPort", "Proxy port", ContextRequirements.FieldType.TEXT, false)); } public byte[][] getAccessKey() { return (byte[][]) getContext().getConfigurationValue(DSN_ACCESS_KEY); } @Override public @Nonnull RegionsAndZones getDataCenterServices() { return new RegionsAndZones(this); } private transient volatile EC2Provider provider; public @Nonnull EC2Provider getEC2Provider() { if (provider == null) { provider = EC2Provider.valueOf(getProviderName()); } return provider; } public @Nullable String getEc2Url() { ProviderContext ctx = getContext(); String url = getEc2Url(ctx == null ? null : ctx.getRegionId()); if (getEC2Provider().isEucalyptus()) { return url + "/Eucalyptus"; } else { return url; } } public @Nullable String getEc2Url(@Nullable String regionId) { ProviderContext ctx = getContext(); String url; if (regionId == null || regionId.isEmpty()) { return getBootstrapUrls(ctx)[0]; } if (getEC2Provider().isAWS()) { url = (ctx == null ? null : ctx.getCloud().getEndpoint()); if (url != null && url.endsWith("amazonaws.com")) { return "https://ec2." + regionId + ".amazonaws.com"; } return "https://ec2." + regionId + ".amazonaws.com"; } else if (!getEC2Provider().isEucalyptus()) { url = (ctx == null ? null : ctx.getCloud().getEndpoint()); if (url == null) { return null; } if (!url.startsWith("http")) { String cloudUrl = ctx.getCloud().getEndpoint(); if (cloudUrl != null && cloudUrl.startsWith("http:")) { return "http://" + url + "/" + regionId; } return "https://" + url + "/" + regionId; } else { return url + "/" + regionId; } } url = (ctx == null ? null : ctx.getCloud().getEndpoint()); if (url == null) { return null; } if (!url.startsWith("http")) { String cloudUrl = ctx.getCloud().getEndpoint(); if (cloudUrl != null && cloudUrl.startsWith("http:")) { return "http://" + url; } return "https://" + url; } else { return url; } } public String getGlacierUrl() throws InternalException, CloudException { ProviderContext ctx = getContext(); String regionId = ctx.getRegionId(); return "https://glacier." + regionId + ".amazonaws.com/-/"; } public String getAutoScaleVersion() { return "2011-01-01"; } public String getCloudWatchVersion() { return "2010-08-01"; } public String getEc2Version() { if (getEC2Provider().isAWS()) { return "2014-05-01"; } else if (getEC2Provider().isEucalyptus()) { return "2010-11-15"; } else if (getEC2Provider().isOpenStack()) { return "2009-11-30"; } return "2012-07-20"; } public String getElbVersion() { return "2012-06-01"; } public String getRdsVersion() { return "2014-09-01"; } public String getRoute53Version() { return "2012-12-12"; } public String getSdbVersion() { return "2009-04-15"; } public String getSnsVersion() { return "2010-03-31"; } public String getSqsVersion() { return "2009-02-01"; } @Override public AWSIdentityServices getIdentityServices() { if (getEC2Provider().isStorage()) { return null; } return new AWSIdentityServices(this); } @Override public EC2NetworkServices getNetworkServices() { if (getEC2Provider().isStorage()) { return null; } return new EC2NetworkServices(this); } @Override public @Nullable AWSPlatformServices getPlatformServices() { EC2Provider p = getEC2Provider(); if (p.isAWS() || p.isEnStratus()) { return new AWSPlatformServices(this); } return null; } @Override public @Nonnull String getProviderName() { ProviderContext ctx = getContext(); String name = (ctx == null ? null : ctx.getCloud().getProviderName()); return ((name == null) ? EC2Provider.AWS.getName() : name); } public @Nullable String getProxyHost() { ProviderContext ctx = getContext(); if (ctx == null) { return null; } Properties props = ctx.getCustomProperties(); return (props == null ? null : props.getProperty("proxyHost")); } public int getProxyPort() { ProviderContext ctx = getContext(); if (ctx == null) { return -1; } Properties props = ctx.getCustomProperties(); if (props == null) { return -1; } String port = props.getProperty("proxyPort"); if (port != null) { return Integer.parseInt(port); } return -1; } @Override public @Nonnull AWSCloudStorageServices getStorageServices() { return new AWSCloudStorageServices(this); } public Map<String, String> getStandardParameters(ProviderContext ctx, String action) throws InternalException { return getStandardParameters(ctx, action, getEc2Version()); } public Map<String, String> getStandardParameters(ProviderContext ctx, String action, String version) throws InternalException { Map<String, String> parameters = new HashMap<String, String>(); parameters.put(P_ACTION, action); // parameters.put(P_SIGNATURE_VERSION, SIGNATURE_V4); // try { // byte[][] keys = getAccessKey(); // // parameters.put(P_ACCESS, new String(keys[0], "utf-8")); // } catch( UnsupportedEncodingException e ) { // logger.error(e); // e.printStackTrace(); // throw new InternalException(e); // } // parameters.put(P_SIGNATURE_METHOD, EC2_ALGORITHM); // parameters.put(P_TIMESTAMP, getTimestamp(System.currentTimeMillis(), true)); parameters.put(P_VERSION, version); return parameters; } private @Nonnull Map<String, String> getElbParameters(@Nonnull ProviderContext ctx, @Nonnull String action) throws InternalException { Map<String, String> parameters = getStandardParameters(ctx, action); parameters.put(P_VERSION, getElbVersion()); return parameters; } public Map<String, String> getStandardCloudWatchParameters(ProviderContext ctx, String action) throws InternalException { Map<String, String> parameters = getStandardParameters(ctx, action); parameters.put(P_VERSION, getCloudWatchVersion()); return parameters; } public Map<String, String> getStandardRdsParameters(ProviderContext ctx, String action) throws InternalException { Map<String, String> parameters = getStandardParameters(ctx, action); parameters.put(P_VERSION, getRdsVersion()); return parameters; } public Map<String, String> getStandardSimpleDBParameters(ProviderContext ctx, String action) throws InternalException { Map<String, String> parameters = getStandardParameters(ctx, action); parameters.put(P_VERSION, getSdbVersion()); return parameters; } public Map<String, String> getStandardSnsParameters(ProviderContext ctx, String action) throws InternalException { Map<String, String> parameters = getStandardParameters(ctx, action); parameters.put(P_VERSION, getSnsVersion()); return parameters; } public Map<String, String> getStandardSqsParameters(ProviderContext ctx, String action) throws InternalException { Map<String, String> parameters = getStandardParameters(ctx, action); parameters.put(P_VERSION, getSqsVersion()); return parameters; } public static void addExtraParameters(Map<String, String> parameters, Map<String, String> extraParameters) { if (extraParameters == null || extraParameters.size() == 0) { return; } if (parameters == null) { parameters = new HashMap<String, String>(); } parameters.putAll(extraParameters); } public static @Nullable Map<String, String> getTagFilterParams(@Nullable Map<String, String> tags) { return getTagFilterParams(tags, 1); } public static @Nullable Map<String, String> getTagFilterParams(@Nullable Map<String, String> tags, int startingFilterIndex) { if (tags == null || tags.size() == 0) { return null; } Map<String, String> filterParameters = new HashMap<String, String>(); int i = startingFilterIndex; for (Map.Entry<String, String> parameter : tags.entrySet()) { addFilterParameters(filterParameters, i, "tag:" + parameter.getKey(), Collections.singletonList(parameter.getValue())); i++; } return filterParameters; } public static void addFilterParameters(Map<String, String> filterParameters, int index, String filterName, Collection<?> filterValues) { if (filterValues == null || filterValues.isEmpty()) { return; } filterParameters.put("Filter." + index + ".Name", filterName); int valueIndex = 0; for (Object filterValue : filterValues) { // filter values must be in lower case filterParameters.put("Filter." + index + ".Value." + valueIndex++, filterValue.toString().toLowerCase()); } } public static void addFilterParameters(Map<String, String> filterParameters, int index, String filterName, Object... filterValues) { if (filterValues == null || filterValues.length == 0) { return; } filterParameters.put("Filter." + index + ".Name", filterName); int valueIndex = 0; for (Object filterValue : filterValues) { // filter values must be in lower case filterParameters.put("Filter." + index + ".Value." + valueIndex++, filterValue.toString().toLowerCase()); } } public @Nonnull String getTimestamp(long timestamp, boolean withMillis) { SimpleDateFormat fmt; if (withMillis) { fmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); } else { fmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); } fmt.setTimeZone(TimeZone.getTimeZone("UTC")); return fmt.format(new Date(timestamp)); } public long parseTime(@Nullable String time) throws CloudException { if (time == null) { return 0L; } SimpleDateFormat fmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); if (time.length() > 0) { try { return fmt.parse(time).getTime(); } catch (ParseException e) { fmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); try { return fmt.parse(time).getTime(); } catch (ParseException encore) { throw new CloudException("Could not parse date: " + time); } } } return 0L; } private String sign(byte[] key, String authString, String algorithm) throws InternalException { try { Mac mac = Mac.getInstance(algorithm); mac.init(new SecretKeySpec(key, algorithm)); return new String(Base64.encodeBase64(mac.doFinal(authString.getBytes("utf-8")))); } catch (NoSuchAlgorithmException e) { logger.error(e); e.printStackTrace(); throw new InternalException(e); } catch (InvalidKeyException e) { logger.error(e); e.printStackTrace(); throw new InternalException(e); } catch (IllegalStateException e) { logger.error(e); e.printStackTrace(); throw new InternalException(e); } catch (UnsupportedEncodingException e) { logger.error(e); e.printStackTrace(); throw new InternalException(e); } } public String signUploadPolicy(String base64Policy) throws InternalException { ProviderContext ctx = getContext(); if (ctx == null) { throw new InternalException("No context for signing the request"); } return sign(getAccessKey()[1], base64Policy, S3_ALGORITHM); } public String getRequestBodyHash(String bodyText) throws InternalException { if (bodyText == null) { // use hash of the empty string return AWSCloud.computeSHA256Hash(""); } else { return AWSCloud.computeSHA256Hash(bodyText); } } public String signCloudFront(String accessKey, byte[] secretKey, String dateString) throws InternalException { String signature = sign(secretKey, dateString, CLOUD_FRONT_ALGORITHM); if (getEC2Provider().isStorage() && "google".equalsIgnoreCase(getProviderName())) { return ("GOOG1 " + accessKey + ":" + signature); } else { return ("AWS " + accessKey + ":" + signature); } } public String signEc2(byte[] key, String serviceUrl, Map<String, String> parameters) throws InternalException { return sign(key, buildEc2AuthString("POST", serviceUrl, parameters), EC2_ALGORITHM); } public String signAWS3(String keyId, byte[] key, String dateString) throws InternalException { return ("AWS3-HTTPS AWSAccessKeyId=" + keyId + ",Algorithm=" + EC2_ALGORITHM + ",Signature=" + sign(key, dateString, EC2_ALGORITHM)); } public String signS3(String accessKey, byte[] secretKey, String action, String hash, String contentType, Map<String, String> headers, String bucket, String object) throws InternalException { StringBuilder toSign = new StringBuilder(); toSign.append(action); toSign.append("\n"); if (hash != null) { toSign.append(hash); } toSign.append("\n"); if (contentType != null) { toSign.append(contentType); } toSign.append("\n\n"); ArrayList<String> keys = new ArrayList<String>(); keys.addAll(headers.keySet()); Collections.sort(keys); for (String hkey : keys) { if (hkey.startsWith("x-amz") || (getEC2Provider().isStorage() && hkey.startsWith("x-goog"))) { String val = headers.get(hkey); if (val != null) { toSign.append(hkey); toSign.append(":"); toSign.append(headers.get(hkey).trim()); toSign.append("\n"); } } } toSign.append("/"); if (getEC2Provider().isEucalyptus()) { toSign.append("services/Walrus/"); } if (bucket != null) { toSign.append(bucket); toSign.append("/"); } if (object != null) { toSign.append(object.toLowerCase()); } String signature = sign(secretKey, toSign.toString(), S3_ALGORITHM); if (getEC2Provider().isStorage() && "google".equalsIgnoreCase(getProviderName())) { return ("GOOG1 " + accessKey + ":" + signature); } else { return ("AWS " + accessKey + ":" + signature); } } /** * Generates an AWS v4 signature authorization string * * @param accessKey Amazon credential * @param secretKey Amazon credential * @param action the HTTP method (GET, POST, etc) * @param url the full URL for the request, including any query parameters * @param serviceId the canonical name of the service targeted in the request (e.g. "glacier") * @param headers map of headers of request. MUST include x-amz-date or date header. * @param bodyHash a hex-encoded sha256 hash of the body of the request * @return a string suitable for including as the HTTP Authorization header * @throws InternalException */ public String getV4Authorization(String accessKey, String secretKey, String action, String url, String serviceId, Map<String, String> headers, String bodyHash) throws InternalException { serviceId = serviceId.toLowerCase(); String regionId = "us-east-1"; // default for IAM // if( ctx != null && ctx.getRegionId() != null && !ctx.getRegionId().isEmpty() && // !serviceId.equalsIgnoreCase(IAMMethod.SERVICE_ID) ) { // regionId = getContext().getRegionId(); // } else { String host = url.replaceAll("https?:\\/\\/", ""); if (host.indexOf('/') > 0) { host = host.substring(0, host.indexOf('/', 1)); } if (!IAMMethod.SERVICE_ID.equalsIgnoreCase(serviceId)) { String[] urlParts = host.split("\\."); // everywhere except s3 and iam this is: service.region.amazonaws.com regionId = urlParts[urlParts.length - 3]; if (regionId.startsWith("s3-")) { regionId = regionId.substring(3); } } String amzDate = extractV4Date(headers); String credentialScope = getV4CredentialScope(amzDate, regionId, serviceId); String signedHeaders = getV4SignedHeaders(headers); String signature = signV4(secretKey, action, url, regionId, serviceId, headers, bodyHash); return V4_ALGORITHM + " " + "Credential=" + accessKey + "/" + credentialScope + ", " + "SignedHeaders=" + signedHeaders + ", " + "Signature=" + signature; } private String signV4(String secretKey, String action, String serviceUrl, String regionId, String serviceId, Map<String, String> headers, String bodyHash) throws InternalException { final String canonicalRequest = getV4CanonicalRequest(action, serviceUrl, headers, bodyHash); String amzDate = extractV4Date(headers); final String stringToSign = getV4StringToSign(amzDate, regionId, serviceId, canonicalRequest); // signature uses YYYYMMDD String dateStamp = amzDate.substring(0, 8); final byte[] signingKey = getV4SigningKey(secretKey, dateStamp, regionId, serviceId); return new String(Hex.encodeHex(HmacSHA256(stringToSign, signingKey), true)); } private String extractV4Date(Map<String, String> headers) throws InternalException { Map<String, String> lower = new HashMap<String, String>(); for (Map.Entry<String, String> entry : headers.entrySet()) { lower.put(entry.getKey().toLowerCase(), entry.getValue()); } String amzDate = headers.get(P_AWS_DATE); // expecting YYYYMMDDTHHMMSSZ if (amzDate != null) { if (amzDate.length() != 16) { throw new InternalException("request has invalid " + P_AWS_DATE); } return amzDate; } String date = lower.get("date"); if (date == null) { throw new InternalException("request is missing date header"); } SimpleDateFormat parser = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz"); try { return getV4HeaderDate(parser.parse(date)); } catch (ParseException e) { throw new InternalException("request has invalid date header format"); } } private byte[] getV4SigningKey(String secretKey, String dateStamp, String regionId, String serviceId) throws InternalException { byte[] withSecret = ("AWS4" + secretKey).getBytes(); byte[] withDate = HmacSHA256(dateStamp, withSecret); byte[] withRegion = HmacSHA256(regionId, withDate); byte[] withService = HmacSHA256(serviceId, withRegion); return HmacSHA256("aws4_request", withService); } private String getV4StringToSign(String dateStamp, String regionId, String serviceId, String canonicalRequest) throws InternalException { return V4_ALGORITHM + "\n" + dateStamp + "\n" + getV4CredentialScope(dateStamp, regionId, serviceId) + "\n" + computeSHA256Hash(canonicalRequest); } private String getV4CredentialScope(String dateStamp, String regionId, String serviceId) { return dateStamp.substring(0, 8) + "/" + regionId + "/" + serviceId + "/" + V4_TERMINATION; } private String getV4CanonicalRequest(String action, String serviceUrl, Map<String, String> headers, String bodyHash) throws InternalException { /* CanonicalRequest = HTTPRequestMethod + '\n' + CanonicalURI + '\n' + CanonicalQueryString + '\n' + CanonicalHeaders + '\n' + SignedHeaders + '\n' + HexEncode(Hash(Payload)) */ final URI endpoint; try { endpoint = new URI(serviceUrl.replace(" ", "%20")).normalize(); } catch (URISyntaxException e) { throw new InternalException(e); } final StringBuilder s = new StringBuilder(); s.append(action.toUpperCase()).append("\n"); String path = endpoint.getPath(); if (path == null || path.length() == 0) { path = "/"; } s.append(encode(path, true)).append("\n"); s.append(getV4CanonicalQueryString(endpoint)).append("\n"); List<String> sortedHeaders = new ArrayList<String>(); sortedHeaders.addAll(headers.keySet()); Collections.sort(sortedHeaders, String.CASE_INSENSITIVE_ORDER); for (String header : sortedHeaders) { String value = headers.get(header).trim().replaceAll("\\s+", " "); header = header.toLowerCase().replaceAll("\\s+", " "); s.append(header).append(":").append(value).append("\n"); } s.append("\n").append(getV4SignedHeaders(headers)).append("\n").append(bodyHash); return s.toString(); } private String getV4CanonicalQueryString(URI endpoint) throws InternalException { // parse query params and translate to another form of tuple that is comparable on both key and value List<NameValuePair> parsedParams = URLEncodedUtils.parse(endpoint, "UTF-8"); List<KeyValuePair> queryParams = new ArrayList<KeyValuePair>(parsedParams.size()); for (NameValuePair param : parsedParams) { String key = encode(param.getName(), false); String value = param.getValue() != null ? encode(param.getValue(), false) : ""; queryParams.add(new KeyValuePair(key, value)); } // sort query parameters by key, then value Collections.sort(queryParams); StringBuilder sb = new StringBuilder(); for (KeyValuePair pair : queryParams) { if (sb.length() > 0) { sb.append("&"); } sb.append(pair.getKey()).append("=").append(pair.getValue()); } return sb.toString(); } private String getV4SignedHeaders(Map<String, String> headers) { // move to set to lower case and remove dupes Set<String> sorted = new TreeSet<String>(); for (String header : headers.keySet()) { sorted.add(header.toLowerCase()); } StringBuilder sb = new StringBuilder(); for (String header : sorted) { if (sb.length() > 0) { sb.append(";"); } sb.append(header.toLowerCase()); } return sb.toString(); } public String getV4HeaderDate(Date date) { SimpleDateFormat fmt = new SimpleDateFormat("yyyyMMdd'T'HHmmss'Z'"); Calendar cal = Calendar.getInstance(new SimpleTimeZone(0, "GMT")); fmt.setCalendar(cal); if (date == null) { return fmt.format(new Date()); } else { return fmt.format(date); } } @Override public String testContext() { APITrace.begin(this, "Cloud.testContext"); try { ProviderContext ctx = getContext(); if (ctx == null) { logger.warn("No context exists for testing"); return null; } try { ComputeServices compute = getComputeServices(); if (compute != null) { VirtualMachineSupport support = compute.getVirtualMachineSupport(); if (support == null || !support.isSubscribed()) { logger.warn("Not subscribed to virtual machine support"); return null; } String actualAccountNumber = getOwnerId(); // Return actual account number as the number provided in configuration // may have been incorrect if (actualAccountNumber != null) { return actualAccountNumber; } } else { StorageServices storage = getStorageServices(); BlobStoreSupport support = storage.getOnlineStorageSupport(); if (support == null || !support.isSubscribed()) { logger.warn("No subscribed to storage services"); return null; } } } catch (Throwable t) { logger.warn("Unable to connect to AWS for " + ctx.getAccountNumber() + ": " + t.getMessage()); return null; } return ctx.getAccountNumber(); } finally { APITrace.end(); } } /** * Retrieve current account number using DescribeSecurityGroups. May not always be reliable but is better than * nothing. * * @return current account number or null if not found */ private String getOwnerId() { APITrace.begin(this, "AWSCloud.getOwnerId"); try { ProviderContext ctx = getContext(); if (ctx == null) { return null; } Map<String, String> parameters = getStandardParameters(getContext(), EC2Method.DESCRIBE_SECURITY_GROUPS); EC2Method method; NodeList blocks; Document doc; method = new EC2Method(EC2Method.SERVICE_ID, this, parameters); try { doc = method.invoke(); } catch (EC2Exception e) { logger.error(e.getSummary()); throw new CloudException(e); } blocks = doc.getElementsByTagName("securityGroupInfo"); for (int i = 0; i < blocks.getLength(); i++) { NodeList items = blocks.item(i).getChildNodes(); for (int j = 0; j < items.getLength(); j++) { Node item = items.item(j); if (item.getNodeName().equals("item")) { NodeList attrs = item.getChildNodes(); for (int k = 0; k < attrs.getLength(); k++) { Node attr = attrs.item(k); if (attr.getNodeName().equals("ownerId")) { return attr.getFirstChild().getNodeValue().trim(); } } } } } return null; } catch (InternalException e) { } catch (CloudException e) { } finally { APITrace.end(); } // Couldn't get the number for some reason return null; } public void setTags(@Nonnull Node attr, @Nonnull Taggable item) { if (attr.hasChildNodes()) { NodeList tags = attr.getChildNodes(); for (int j = 0; j < tags.getLength(); j++) { Tag t = toTag(tags.item(j)); if (t != null && t.getValue() != null) { item.setTag(t.getKey(), t.getValue()); } } } } /** * Gets the epoch form of the text value of the provided node. * * @param node the node to extact the value from * @return the epoch time * @throws CloudException */ public static long getTimestampValue(Node node) throws CloudException { SimpleDateFormat fmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); fmt.setTimeZone(TimeZone.getTimeZone("UTC")); String value = getTextValue(node); try { return fmt.parse(value).getTime(); } catch (ParseException e) { logger.error(e); e.printStackTrace(); throw new CloudException(e); } } /** * Returns the text from the given node. * * @param node the node to extract the value from * @return the text from the node */ static public String getTextValue(Node node) { if (node.getChildNodes().getLength() == 0) { return null; } return node.getFirstChild().getNodeValue(); } /** * Returns the boolean value of the given node. * * @param node the node to extract the value from * @return the boolean value of the node */ static public boolean getBooleanValue(Node node) { return Boolean.valueOf(getTextValue(node)); } /** * Returns the int value of the given node. * * @param node the node to extract the value from * @return the int value of the given node */ public static int getIntValue(Node node) { return Integer.valueOf(getTextValue(node)); } /** * Returns the double value of the given node. * * @param node the node to extract the value from * @return the double value of the given node */ public static double getDoubleValue(Node node) { return Double.valueOf(getTextValue(node)); } /** * Returns the float value of the given node. * * @param node the node to extract the value from * @return the float value of the given node */ public static float getFloatValue(Node node) { return Float.valueOf(getTextValue(node)); } /** * Helper method for adding indexed member parameters, e.g. <i>AlarmNames.member.N</i>. Will overwrite existing * parameters if present. Assumes indexing starts at 1. * * @param parameters the existing parameters map to add to * @param prefix the prefix value for each parameter key * @param values the values to add */ public static void addIndexedParameters(@Nonnull Map<String, String> parameters, @Nonnull String prefix, String... values) { if (values == null || values.length == 0) { return; } int i = 1; if (!prefix.endsWith(".")) { prefix += "."; } for (String value : values) { parameters.put(String.format("%s%d", prefix, i), value); i++; } } /** * Helper method for adding indexed member parameters, e.g. <i>AlarmNames.member.N</i>. Will overwrite existing * parameters if present. Assumes indexing starts at 1. * * @param parameters the existing parameters map to add to * @param prefix the prefix value for each parameter key * @param extraParameters the values to add */ public static void addIndexedParameters(@Nonnull Map<String, String> parameters, @Nonnull String prefix, Map<String, String> extraParameters) { if (extraParameters == null || extraParameters.size() == 0) { return; } int i = 1; for (Map.Entry<String, String> entry : extraParameters.entrySet()) { parameters.put(prefix + i + ".Name", entry.getKey()); if (entry.getValue() != null) { parameters.put(prefix + i + ".Value", entry.getValue()); } i++; } } /** * Puts the given key/value into the given map only if the value is not null. * * @param parameters the map to add to * @param key the key of the value * @param value the value to add if not null */ public static void addValueIfNotNull(@Nonnull Map<String, String> parameters, @Nonnull String key, Object value) { if (value == null) { return; } parameters.put(key, value.toString()); } public @Nonnull HttpClient getClient() throws InternalException { return getClient(false); } public @Nonnull HttpClient getClient(boolean multipart) throws InternalException { ProviderContext ctx = getContext(); if (ctx == null) { throw new InternalException("No context was specified for this request"); } final HttpParams params = new BasicHttpParams(); HttpProtocolParams.setVersion(params, HttpVersion.HTTP_1_1); if (!multipart) { HttpProtocolParams.setContentCharset(params, Consts.UTF_8.toString()); } HttpProtocolParams.setUserAgent(params, "Dasein Cloud"); Properties p = ctx.getCustomProperties(); if (p != null) { String proxyHost = p.getProperty("proxyHost"); String proxyPortStr = p.getProperty("proxyPort"); int proxyPort = 0; if (proxyPortStr != null) { proxyPort = Integer.parseInt(proxyPortStr); } if (proxyHost != null && proxyHost.length() > 0 && proxyPort > 0) { params.setParameter(ConnRoutePNames.DEFAULT_PROXY, new HttpHost(proxyHost, proxyPort)); } } DefaultHttpClient httpClient = new DefaultHttpClient(params); httpClient.addRequestInterceptor(new HttpRequestInterceptor() { public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { if (!request.containsHeader("Accept-Encoding")) { request.addHeader("Accept-Encoding", "gzip"); } request.setParams(params); } }); httpClient.addResponseInterceptor(new HttpResponseInterceptor() { public void process(final HttpResponse response, final HttpContext context) throws HttpException, IOException { HttpEntity entity = response.getEntity(); if (entity != null) { Header header = entity.getContentEncoding(); if (header != null) { for (HeaderElement codec : header.getElements()) { if (codec.getName().equalsIgnoreCase("gzip")) { response.setEntity(new GzipDecompressingEntity(response.getEntity())); break; } } } } } }); return httpClient; } /** * DEBUG_AWS should be specified as system properties, otherwise return false * * @return DEBUG_AWS properties value, or false if not specified */ public boolean isDebug() { return Boolean.valueOf(System.getProperties().getProperty("DEBUG_AWS", "false")); } }