Java tutorial
/* * Copyright 2014 EMC Corporation. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. * A copy of the License is located at * * http://www.apache.org/licenses/LICENSE-2.0.txt * * or in the "license" file accompanying this file. This file is distributed * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ package com.emc.esu.api.rest; import com.emc.esu.api.*; import com.emc.esu.api.Grantee.GRANT_TYPE; import com.emc.util.HttpUtil; import org.apache.commons.codec.binary.Base64; import org.apache.log4j.Logger; import org.jdom.Document; import org.jdom.Element; import org.jdom.JDOMException; import org.jdom.Namespace; import org.jdom.input.SAXBuilder; import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; import java.io.*; import java.net.*; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; /** * Encapsulates common REST API functionality that is independant of * the transport layer, e.g. signature generation and getShareableUrl. * @author Jason Cwik */ public abstract class AbstractEsuRestApi implements EsuApi { private static final DateFormat HEADER_FORMAT = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z", Locale.ENGLISH); private static final String ISO8601_FORMAT = "yyyy-MM-dd'T'HH:mm:ss'Z'"; private static final Pattern OBJECTID_EXTRACTOR = Pattern.compile("/\\w+/objects/([0-9a-f]{44,})"); private static final Logger l4j = Logger.getLogger(AbstractEsuRestApi.class); protected String host; protected int port; protected String uid; protected byte[] secret; protected String context = "/rest"; protected String proto; protected boolean unicodeEnabled = false; protected boolean readChecksum; private long serverOffset; /** * Creates a new AbstractEsuRestApi * @param host the host running the web services * @param port the port number, e.g. 80 or 443 * @param uid the web service UID * @param sharedSecret the UID's shared secret key */ public AbstractEsuRestApi(String host, int port, String uid, String sharedSecret) { try { this.secret = Base64.decodeBase64(sharedSecret.getBytes("UTF-8")); } catch (UnsupportedEncodingException e) { throw new EsuException("Could not decode shared secret", e); } this.host = host; this.uid = uid; this.port = port; if (port == 443) { proto = "https"; } else { proto = "http"; } } /** * Gets the context root of the REST api. By default this is /rest. * * @return the context */ public String getContext() { return context; } /** * Overrides the default context root of the REST api. * * @param context the context to set */ public void setContext(String context) { this.context = context; } /** * Returns the protocol being used (http or https). * * @return the proto */ public String getProtocol() { return proto; } /** * Overrides the protocol selection. By default, https will be used for port * 443. Http will be used otherwise * * @param proto the proto to set */ public void setProtocol(String proto) { this.proto = proto; } /** * Creates a new object in the cloud. * * @param acl Access control list for the new object. May be null to use a * default ACL * @param metadata Metadata for the new object. May be null for no metadata. * @param data The initial contents of the object. May be appended to later. * May be null to create an object with no content. * @param mimeType the MIME type of the content. Optional, may be null. If * data is non-null and mimeType is null, the MIME type will * default to application/octet-stream. * @return Identifier of the newly created object. * @throws EsuException if the request fails. */ public ObjectId createObject(Acl acl, MetadataList metadata, byte[] data, String mimeType) { return createObjectFromSegment(acl, metadata, data == null ? null : new BufferSegment(data), mimeType, null); } /** * Creates a new object in the cloud. * * @param acl Access control list for the new object. May be null to use a * default ACL * @param metadata Metadata for the new object. May be null for no metadata. * @param data The initial contents of the object. May be appended to later. * May be null to create an object with no content. * @param mimeType the MIME type of the content. Optional, may be null. If * data is non-null and mimeType is null, the MIME type will * default to application/octet-stream. * @param checksum if not null, use the Checksum object to compute * the checksum for the create object request. If appending * to the object with subsequent requests, use the same * checksum object for each request. * @return Identifier of the newly created object. * @throws EsuException if the request fails. */ public ObjectId createObject(Acl acl, MetadataList metadata, byte[] data, String mimeType, Checksum checksum) { return createObjectFromSegment(acl, metadata, data == null ? null : new BufferSegment(data), mimeType, checksum); } /** * Creates a new object in the cloud using a BufferSegment. * * @param acl Access control list for the new object. May be null to use a * default ACL * @param metadata Metadata for the new object. May be null for no metadata. * @param data The initial contents of the object. May be appended to later. * May be null to create an object with no content. * @param mimeType the MIME type of the content. Optional, may be null. If * data is non-null and mimeType is null, the MIME type will * default to application/octet-stream. * @return Identifier of the newly created object. * @throws EsuException if the request fails. */ public ObjectId createObjectFromSegment(Acl acl, MetadataList metadata, BufferSegment data, String mimeType) { return createObjectFromSegment(acl, metadata, data, mimeType, null); } /** * Creates a new object in the cloud on the specified path. * * @param path The path to create the object on. * @param acl Access control list for the new object. May be null to use a * default ACL * @param metadata Metadata for the new object. May be null for no metadata. * @param data The initial contents of the object. May be appended to later. * May be null to create an object with no content. * @param mimeType the MIME type of the content. Optional, may be null. If * data is non-null and mimeType is null, the MIME type will * default to application/octet-stream. * @return the ObjectId of the newly-created object for references by ID. * @throws EsuException if the request fails. */ public ObjectId createObjectOnPath(ObjectPath path, Acl acl, MetadataList metadata, byte[] data, String mimeType) { return createObjectFromSegmentOnPath(path, acl, metadata, data == null ? null : new BufferSegment(data), mimeType, null); } /** * Creates a new object in the cloud on the specified path. * * @param path The path to create the object on. * @param acl Access control list for the new object. May be null to use a * default ACL * @param metadata Metadata for the new object. May be null for no metadata. * @param data The initial contents of the object. May be appended to later. * May be null to create an object with no content. * @param mimeType the MIME type of the content. Optional, may be null. If * data is non-null and mimeType is null, the MIME type will * default to application/octet-stream. * @param checksum if not null, use the Checksum object to compute * the checksum for the create object request. If appending * to the object with subsequent requests, use the same * checksum object for each request. * @return the ObjectId of the newly-created object for references by ID. * @throws EsuException if the request fails. */ public ObjectId createObjectOnPath(ObjectPath path, Acl acl, MetadataList metadata, byte[] data, String mimeType, Checksum checksum) { return createObjectFromSegmentOnPath(path, acl, metadata, data == null ? null : new BufferSegment(data), mimeType, checksum); } /** * Creates a new object in the cloud using a BufferSegment on the given * path. * * @param path the path to create the object on. * @param acl Access control list for the new object. May be null to use a * default ACL * @param metadata Metadata for the new object. May be null for no metadata. * @param data The initial contents of the object. May be appended to later. * May be null to create an object with no content. * @param mimeType the MIME type of the content. Optional, may be null. If * data is non-null and mimeType is null, the MIME type will * default to application/octet-stream. * @return the ObjectId of the newly-created object for references by ID. * @throws EsuException if the request fails. */ public ObjectId createObjectFromSegmentOnPath(ObjectPath path, Acl acl, MetadataList metadata, BufferSegment data, String mimeType) { return createObjectFromSegmentOnPath(path, acl, metadata, data, mimeType, null); } /** * Updates an object in the cloud and optionally its metadata and ACL. * * @param id The ID of the object to update * @param acl Access control list for the new object. Optional, default is * NULL to leave the ACL unchanged. * @param metadata Metadata list for the new object. Optional, default is * NULL for no changes to the metadata. * @param data The new contents of the object. May be appended to later. * Optional, default is NULL (no content changes). * @param extent portion of the object to update. May be null to indicate * the whole object is to be replaced. If not null, the extent * size must match the data size. * @param mimeType the MIME type of the content. Optional, may be null. If * data is non-null and mimeType is null, the MIME type will * default to application/octet-stream. * @throws EsuException if the request fails. */ public void updateObject(Identifier id, Acl acl, MetadataList metadata, Extent extent, byte[] data, String mimeType) { updateObjectFromSegment(id, acl, metadata, extent, data == null ? null : new BufferSegment(data), mimeType, null); } /** * Updates an object in the cloud and optionally its metadata and ACL. * * @param id The ID of the object to update * @param acl Access control list for the new object. Optional, default is * NULL to leave the ACL unchanged. * @param metadata Metadata list for the new object. Optional, default is * NULL for no changes to the metadata. * @param data The new contents of the object. May be appended to later. * Optional, default is NULL (no content changes). * @param extent portion of the object to update. May be null to indicate * the whole object is to be replaced. If not null, the extent * size must match the data size. * @param mimeType the MIME type of the content. Optional, may be null. If * data is non-null and mimeType is null, the MIME type will * default to application/octet-stream. * @param checksum if not null, use the Checksum object to compute * the checksum for the update object request. If appending * to the object with subsequent requests, use the same * checksum object for each request. * @throws EsuException if the request fails. */ public void updateObject(Identifier id, Acl acl, MetadataList metadata, Extent extent, byte[] data, String mimeType, Checksum checksum) { updateObjectFromSegment(id, acl, metadata, extent, data == null ? null : new BufferSegment(data), mimeType, checksum); } /** * Updates an object in the cloud and optionally its metadata and ACL. * * @param id The ID of the object to update * @param acl Access control list for the new object. Optional, default is * NULL to leave the ACL unchanged. * @param metadata Metadata list for the new object. Optional, default is * NULL for no changes to the metadata. * @param data The new contents of the object. May be appended to later. * Optional, default is NULL (no content changes). * @param extent portion of the object to update. May be null to indicate * the whole object is to be replaced. If not null, the extent * size must match the data size. * @param mimeType the MIME type of the content. Optional, may be null. If * data is non-null and mimeType is null, the MIME type will * default to application/octet-stream. * @throws EsuException if the request fails. */ public void updateObjectFromSegment(Identifier id, Acl acl, MetadataList metadata, Extent extent, BufferSegment data, String mimeType) { updateObjectFromSegment(id, acl, metadata, extent, data, mimeType, null); } /** * Reads an object's content. * * @param id the identifier of the object whose content to read. * @param extent the portion of the object data to read. Optional. Default * is null to read the entire object. * @param buffer the buffer to use to read the extent. Must be large enough * to read the response or an error will be thrown. If null, a * buffer will be allocated to hold the response data. If you * pass a buffer that is larger than the extent, only * extent.getSize() bytes will be valid. * @return the object data read as a byte array. */ public byte[] readObject(Identifier id, Extent extent, byte[] buffer) { return readObject(id, extent, buffer, null); } /** * Lists all objects with the given tag. * * @param tag the tag to search for * @return The list of objects with the given tag. If no objects are found * the array will be empty. * @throws EsuException if no objects are found (code 1003) */ public List<Identifier> listObjects(MetadataTag tag) { return filterIdList(listObjects(tag.getName(), null)); } /** * Lists all objects with the given tag. * * @param tag the tag to search for * @param options the options for listing the objects * @return The list of objects with the given tag. If no objects are found * the array will be empty. * @throws EsuException if no objects are found (code 1003) */ public List<ObjectResult> listObjects(MetadataTag tag, ListOptions options) { return listObjects(tag.getName(), options); } /** * Lists all objects with the given tag. * * @param tag the tag to search for * @return The list of objects with the given tag. If no objects are found * the array will be empty. * @throws EsuException if no objects are found (code 1003) */ public List<Identifier> listObjects(String tag) { return filterIdList(listObjects(tag, null)); } /** * Lists all objects with the given tag and returns both their IDs and their * metadata. * * @param tag the tag to search for * @return The list of objects with the given tag. If no objects are found * the array will be empty. */ public List<ObjectResult> listObjectsWithMetadata(MetadataTag tag) { return listObjectsWithMetadata(tag.getName()); } /** * Lists all objects with the given tag and returns both their IDs and their * metadata. * * @param tag the tag to search for * @return The list of objects with the given tag. If no objects are found * the array will be empty. */ public List<ObjectResult> listObjectsWithMetadata(String tag) { ListOptions options = new ListOptions(); options.setIncludeMetadata(true); return listObjects(tag, options); } /** * Lists the contents of a directory. * @param path the path to list. Must be a directory. * @return the directory entries in the directory. * @deprecated Use the version with ListOptions to control the result * count and handle large result sets. */ public List<DirectoryEntry> listDirectory(ObjectPath path) { return listDirectory(path, null); } /** * Generates an HMAC-SHA1 signature of the given input string using the * shared secret key. * @param input the string to sign * @return the HMAC-SHA1 signature in Base64 format * @throws NoSuchAlgorithmException * @throws InvalidKeyException * @throws IllegalStateException * @throws UnsupportedEncodingException */ public String sign(String input) throws NoSuchAlgorithmException, InvalidKeyException, IllegalStateException, UnsupportedEncodingException { // Compute the signature hash l4j.debug("Hashing: \n" + input.toString()); String hashOut = sign(input.getBytes("UTF-8")); l4j.debug("Hash: " + hashOut); return hashOut; } public String sign(byte[] input) throws UnsupportedEncodingException, InvalidKeyException, NoSuchAlgorithmException { Mac mac = Mac.getInstance("HmacSHA1"); SecretKeySpec key = new SecretKeySpec(secret, "HmacSHA1"); mac.init(key); byte[] hashData = mac.doFinal(input); // Encode the hash in Base64. return new String(Base64.encodeBase64(hashData), "UTF-8"); } /** * An Atmos user (UID) can construct a pre-authenticated URL to an * object, which may then be used by anyone to retrieve the * object (e.g., through a browser). This allows an Atmos user * to let a non-Atmos user download a specific object. The * entire object/file is read. * @param id the object to generate the URL for * @param expiration the expiration date of the URL * @return a URL that can be used to share the object's content */ public URL getShareableUrl(Identifier id, Date expiration, String disposition) { try { String resource = getResourcePath(context, id); StringBuffer sb = new StringBuffer(); sb.append("GET\n"); sb.append(resource.toLowerCase() + "\n"); sb.append(uid + "\n"); sb.append("" + (expiration.getTime() / 1000)); if (disposition != null) { sb.append("\n" + disposition); } String signature = sign(sb.toString()); String query = "uid=" + encodeUtf8(uid) + "&expires=" + (expiration.getTime() / 1000) + "&signature=" + encodeUtf8(signature); if (disposition != null) { disposition = encodeUtf8(disposition); query += "&disposition=" + disposition; } // We do this a little strangely here. Technically, the trailing "=" in the Base-64 signature // should be encoded since it's a "reserved" character. Atmos 1.2 is strict about this, but // 1.3 relaxes the rules a bit. Most URL generators (java.net.URI included) don't have facilities // to break down the query components and encode them individually. Therefore, we encode the // query ourselves here and append it to the generated URL. This will then work with both // 1.2 and 1.3. URL u = buildUrl(resource, null); u = new URL(u.toString() + "?" + query); l4j.debug("URL: " + u); return u; } catch (UnsupportedEncodingException e) { throw new EsuException("Unsupported encoding", e); } catch (InvalidKeyException e) { throw new EsuException("Invalid secret key", e); } catch (NoSuchAlgorithmException e) { throw new EsuException("Missing signature algorithm", e); } catch (IllegalStateException e) { throw new EsuException("Error signing request", e); } catch (MalformedURLException e) { throw new EsuException("Invalid URL format", e); } catch (URISyntaxException e) { throw new EsuException("Invalid URL", e); } } /** * An Atmos user (UID) can construct a pre-authenticated URL to an * object, which may then be used by anyone to retrieve the * object (e.g., through a browser). This allows an Atmos user * to let a non-Atmos user download a specific object. The * entire object/file is read. * @param id the object to generate the URL for * @param expiration the expiration date of the URL * @return a URL that can be used to share the object's content */ public URL getShareableUrl(Identifier id, Date expiration) { return getShareableUrl(id, expiration, null); } /** * Gets the appropriate resource path depending on identifier * type. */ protected String getResourcePath(String ctx, Identifier id) { if (id instanceof ObjectId) { return ctx + "/objects/" + id; } else { return ctx + "/namespace" + id; } } /** * Builds a new URL to the given resource * @throws URISyntaxException * @throws MalformedURLException */ protected URL buildUrl(String resource, String query) throws URISyntaxException, MalformedURLException { int uriport = 0; if ("http".equals(proto) && port == 80) { // Default port uriport = -1; } else if ("https".equals(proto) && port == 443) { uriport = -1; } else { uriport = port; } URI uri = new URI(proto, null, host, uriport, resource, query, null); l4j.debug("URI: " + uri); URL u = new URL(uri.toASCIIString()); l4j.debug("URL: " + u); return u; } /** * Helper method that closes a stream ignoring errors. * @param out the OutputStream to close */ protected void silentClose(OutputStream out) { if (out == null) { return; } try { out.close(); } catch (IOException e) { // ignore } } /** * Parses the given header text and appends to the metadata list * @param meta the metadata list to append to * @param header the metadata header to parse * @param listable true if the header being parsed contains listable metadata. * @throws UnsupportedEncodingException */ protected void readMetadata(MetadataList meta, String header, boolean listable) throws UnsupportedEncodingException { if (header == null) { return; } String[] attrs = header.split(",(?=[^,]+=)"); for (int i = 0; i < attrs.length; i++) { String[] nvpair = attrs[i].split("=", 2); String name = nvpair[0]; String value = nvpair.length > 1 ? nvpair[1] : null; name = name.trim(); if (unicodeEnabled) { name = decodeUtf8(name); value = decodeUtf8(value); } Metadata m = new Metadata(name, value, listable); l4j.debug("Meta: " + m); meta.addMetadata(m); } } /** * Enumerates the given list of metadata tags and sets the x-emc-tags * header. * @param tags the tag list to enumerate * @param headers the HTTP request headers * @throws UnsupportedEncodingException */ protected void processTags(MetadataTags tags, Map<String, String> headers) throws UnsupportedEncodingException { StringBuffer taglist = new StringBuffer(); l4j.debug("Processing " + tags.count() + " metadata tag entries"); if (unicodeEnabled) { headers.put("x-emc-utf8", "true"); } for (Iterator<MetadataTag> i = tags.iterator(); i.hasNext();) { MetadataTag tag = i.next(); if (taglist.length() > 0) { taglist.append(","); } taglist.append(unicodeEnabled ? encodeUtf8(tag.getName()) : tag.getName()); } if (taglist.length() > 0) { headers.put("x-emc-tags", taglist.toString()); } } /** * Parses the value of an ACL response header and builds an ACL * @param acl a reference to the ACL to append to * @param header the acl response header * @param type the type of Grantees in the header (user or group) */ protected void readAcl(Acl acl, String header, GRANT_TYPE type) { l4j.debug("readAcl: " + header); String[] grants = header.split(","); for (int i = 0; i < grants.length; i++) { String[] nvpair = grants[i].split("=", 2); String grantee = nvpair[0]; String permission = nvpair[1]; grantee = grantee.trim(); // Currently, the server returns "FULL" instead of "FULL_CONTROL". // For consistency, change this to value use in the request if ("FULL".equals(permission)) { permission = Permission.FULL_CONTROL; } l4j.debug("grant: " + grantee + "." + permission + " (" + type + ")"); Grantee ge = new Grantee(grantee, type); Grant gr = new Grant(ge, permission); l4j.debug("Grant: " + gr); acl.addGrant(gr); } } /** * Parses an XML response and extracts the list of ObjectIDs. * @param response the response byte array to parse as XML * @return the list of object IDs contained in the response. */ @SuppressWarnings("rawtypes") protected List<ObjectId> parseObjectList(byte[] response) { List<ObjectId> objs = new ArrayList<ObjectId>(); // Use JDOM to parse the XML SAXBuilder sb = new SAXBuilder(); try { Document d = sb.build(new ByteArrayInputStream(response)); // The ObjectID element is part of a namespace so we need to use // the namespace to identify the elements. Namespace esuNs = Namespace.getNamespace("http://www.emc.com/cos/"); List children = d.getRootElement().getChildren("ObjectID", esuNs); l4j.debug("Found " + children.size() + " objects"); for (Iterator i = children.iterator(); i.hasNext();) { Object o = i.next(); if (o instanceof Element) { ObjectId oid = new ObjectId(((Element) o).getText()); l4j.debug(oid.toString()); objs.add(oid); } else { l4j.debug(o + " is not an Element!"); } } } catch (JDOMException e) { throw new EsuException("Error parsing response", e); } catch (IOException e) { throw new EsuException("Error reading response", e); } return objs; } @SuppressWarnings("rawtypes") protected List<Identifier> parseVersionList(byte[] response) { List<Identifier> objs = new ArrayList<Identifier>(); // Use JDOM to parse the XML SAXBuilder sb = new SAXBuilder(); try { Document d = sb.build(new ByteArrayInputStream(response)); // The ObjectID element is part of a namespace so we need to use // the namespace to identify the elements. Namespace esuNs = Namespace.getNamespace("http://www.emc.com/cos/"); List children = d.getRootElement().getChildren("Ver", esuNs); l4j.debug("Found " + children.size() + " objects"); for (Iterator i = children.iterator(); i.hasNext();) { Object o = i.next(); if (o instanceof Element) { Element objectIdElement = (Element) ((Element) o).getChildren("OID", esuNs).get(0); ObjectId oid = new ObjectId(objectIdElement.getText()); l4j.debug(oid.toString()); objs.add(oid); } else { l4j.debug(o + " is not an Element!"); } } } catch (JDOMException e) { throw new EsuException("Error parsing response", e); } catch (IOException e) { throw new EsuException("Error reading response", e); } return objs; } @SuppressWarnings("rawtypes") protected List<Version> parseVersionListLong(byte[] response) { List<Version> objs = new ArrayList<Version>(); DateFormat itimeParser = new SimpleDateFormat(ISO8601_FORMAT); itimeParser.setTimeZone(TimeZone.getTimeZone("UTC")); // Use JDOM to parse the XML SAXBuilder sb = new SAXBuilder(); try { Document d = sb.build(new ByteArrayInputStream(response)); // The ObjectID element is part of a namespace so we need to use // the namespace to identify the elements. Namespace esuNs = Namespace.getNamespace("http://www.emc.com/cos/"); List children = d.getRootElement().getChildren("Ver", esuNs); l4j.debug("Found " + children.size() + " objects"); for (Iterator i = children.iterator(); i.hasNext();) { Object o = i.next(); if (o instanceof Element) { Element e = (Element) o; ObjectId id = new ObjectId(e.getChildText("OID", esuNs)); int versionNumber = Integer.parseInt(e.getChildText("VerNum", esuNs)); String sitime = e.getChildText("itime", esuNs); Date itime = null; try { itime = itimeParser.parse(sitime); } catch (ParseException e1) { throw new EsuException("Could not parse itime: " + sitime, e1); } objs.add(new Version(id, versionNumber, itime)); } else { l4j.debug(o + " is not an Element!"); } } } catch (JDOMException e) { throw new EsuException("Error parsing response", e); } catch (IOException e) { throw new EsuException("Error reading response", e); } return objs; } /** * Parses an XML response and extracts the list of ObjectIDs * and metadata. * @param response the response byte array to parse as XML * @return the list of object IDs contained in the response. */ @SuppressWarnings("rawtypes") protected List<ObjectResult> parseObjectListWithMetadata(byte[] response) { List<ObjectResult> objs = new ArrayList<ObjectResult>(); // Use JDOM to parse the XML SAXBuilder sb = new SAXBuilder(); try { Document d = sb.build(new ByteArrayInputStream(response)); // The ObjectID element is part of a namespace so we need to use // the namespace to identify the elements. Namespace esuNs = Namespace.getNamespace("http://www.emc.com/cos/"); List children = d.getRootElement().getChildren("Object", esuNs); l4j.debug("Found " + children.size() + " objects"); for (Iterator i = children.iterator(); i.hasNext();) { Object o = i.next(); if (o instanceof Element) { Element e = (Element) o; ObjectResult obj = new ObjectResult(); Element objectIdElement = e.getChild("ObjectID", esuNs); ObjectId oid = new ObjectId(objectIdElement.getText()); obj.setId(oid); // next, get metadata Element sMeta = e.getChild("SystemMetadataList", esuNs); Element uMeta = e.getChild("UserMetadataList", esuNs); obj.setMetadata(new MetadataList()); if (sMeta != null) { for (Iterator m = sMeta.getChildren("Metadata", esuNs).iterator(); m.hasNext();) { Element metaElement = (Element) m.next(); String mName = metaElement.getChildText("Name", esuNs); String mValue = metaElement.getChildText("Value", esuNs); obj.getMetadata().addMetadata(new Metadata(mName, mValue, false)); } } if (uMeta != null) { for (Iterator m = uMeta.getChildren("Metadata", esuNs).iterator(); m.hasNext();) { Element metaElement = (Element) m.next(); String mName = metaElement.getChildText("Name", esuNs); String mValue = metaElement.getChildText("Value", esuNs); String mListable = metaElement.getChildText("Listable", esuNs); obj.getMetadata().addMetadata(new Metadata(mName, mValue, "true".equals(mListable))); } } objs.add(obj); } else { l4j.debug(o + " is not an Element!"); } } } catch (JDOMException e) { throw new EsuException("Error parsing response", e); } catch (IOException e) { throw new EsuException("Error reading response", e); } return objs; } @SuppressWarnings("rawtypes") protected List<DirectoryEntry> parseDirectoryListing(byte[] data, ObjectPath basePath) { // Parse List<DirectoryEntry> objs = new ArrayList<DirectoryEntry>(); // Use JDOM to parse the XML SAXBuilder sb = new SAXBuilder(); try { Document d = sb.build(new ByteArrayInputStream(data)); // The ObjectID element is part of a namespace so we need to use // the namespace to identify the elements. Namespace esuNs = Namespace.getNamespace("http://www.emc.com/cos/"); List children = d.getRootElement().getChild("DirectoryList", esuNs).getChildren("DirectoryEntry", esuNs); l4j.debug("Found " + children.size() + " objects"); for (Iterator i = children.iterator(); i.hasNext();) { Object o = i.next(); if (o instanceof Element) { DirectoryEntry de = new DirectoryEntry(); de.setId(new ObjectId(((Element) o).getChildText("ObjectID", esuNs))); String name = ((Element) o).getChildText("Filename", esuNs); String type = ((Element) o).getChildText("FileType", esuNs); name = basePath.toString() + name; if ("directory".equals(type)) { name += "/"; } de.setPath(new ObjectPath(name)); de.setType(type); // next, get metadata Element sMeta = ((Element) o).getChild("SystemMetadataList", esuNs); Element uMeta = ((Element) o).getChild("UserMetadataList", esuNs); if (sMeta != null) { de.setSystemMetadata(new MetadataList()); for (Iterator m = sMeta.getChildren("Metadata", esuNs).iterator(); m.hasNext();) { Element metaElement = (Element) m.next(); String mName = metaElement.getChildText("Name", esuNs); String mValue = metaElement.getChildText("Value", esuNs); de.getSystemMetadata().addMetadata(new Metadata(mName, mValue, false)); } } if (uMeta != null) { de.setUserMetadata(new MetadataList()); for (Iterator m = uMeta.getChildren("Metadata", esuNs).iterator(); m.hasNext();) { Element metaElement = (Element) m.next(); String mName = metaElement.getChildText("Name", esuNs); String mValue = metaElement.getChildText("Value", esuNs); String mListable = metaElement.getChildText("Listable", esuNs); de.getUserMetadata().addMetadata(new Metadata(mName, mValue, "true".equals(mListable))); } } objs.add(de); } else { l4j.debug(o + " is not an Element!"); } } } catch (JDOMException e) { throw new EsuException("Error parsing response", e); } catch (IOException e) { throw new EsuException("Error reading response", e); } return objs; } /** * Parses the given header and appends to the list of metadata tags. * @param tags the list of metadata tags to append to * @param header the header to parse * @param listable true if the metadata tags in the header are listable * @throws UnsupportedEncodingException */ protected void readTags(MetadataTags tags, String header, boolean listable) throws UnsupportedEncodingException { if (header == null) { return; } String[] attrs = header.split(","); for (int i = 0; i < attrs.length; i++) { String attr = attrs[i].trim(); tags.addTag(new MetadataTag(unicodeEnabled ? decodeUtf8(attr) : attr, listable)); } } /** * Iterates through the given metadata and adds the appropriate metadata * headers to the request. * * @param metadata the metadata to add * @param headers the map of request headers. * @throws UnsupportedEncodingException */ protected void processMetadata(MetadataList metadata, Map<String, String> headers) throws UnsupportedEncodingException { StringBuffer listable = new StringBuffer(); StringBuffer nonListable = new StringBuffer(); if (unicodeEnabled) { headers.put("x-emc-utf8", "true"); } l4j.debug("Processing " + metadata.count() + " metadata entries"); for (Iterator<Metadata> i = metadata.iterator(); i.hasNext();) { Metadata meta = i.next(); if (meta.isListable()) { if (listable.length() > 0) { listable.append(", "); } listable.append(formatTag(meta)); } else { if (nonListable.length() > 0) { nonListable.append(", "); } nonListable.append(formatTag(meta)); } } // Only set the headers if there's data if (listable.length() > 0) { headers.put("x-emc-listable-meta", listable.toString()); } if (nonListable.length() > 0) { headers.put("x-emc-meta", nonListable.toString()); } } /** * Formats a tag value for passing in the header. * @throws UnsupportedEncodingException */ protected String formatTag(Metadata meta) throws UnsupportedEncodingException { // strip commas and newlines for now. if (unicodeEnabled) { String name = encodeUtf8(meta.getName()); if (meta.getValue() == null) { return name + "="; } String value = encodeUtf8(meta.getValue()); return name + "=" + value; } else { if (meta.getValue() == null) { return meta.getName() + "="; } String fixed = meta.getValue().replace("\n", ""); fixed = fixed.replace(",", ""); return meta.getName() + "=" + fixed; } } protected String encodeUtf8(String value) throws UnsupportedEncodingException { return HttpUtil.encodeUtf8(value); } protected String decodeUtf8(String value) throws UnsupportedEncodingException { return HttpUtil.decodeUtf8(value); } /** * Enumerates the given ACL and creates the appropriate request headers. * * @param acl the ACL to enumerate * @param headers the set of request headers. */ protected void processAcl(Acl acl, Map<String, String> headers) { StringBuffer userGrants = new StringBuffer(); StringBuffer groupGrants = new StringBuffer(); for (Iterator<Grant> i = acl.iterator(); i.hasNext();) { Grant grant = i.next(); if (grant.getGrantee().getType() == Grantee.GRANT_TYPE.USER) { if (userGrants.length() > 0) { userGrants.append(","); } userGrants.append(grant.toString()); } else { if (groupGrants.length() > 0) { groupGrants.append(","); } groupGrants.append(grant.toString()); } } headers.put("x-emc-useracl", userGrants.toString()); headers.put("x-emc-groupacl", groupGrants.toString()); } /** * Condenses consecutive spaces into one. */ protected String normalizeSpace(String str) { int length = str.length(); while (true) { str = str.replace(" ", " "); if (str.length() == length) { // unchanged break; } length = str.length(); } // Strip any trailing space while (str.endsWith(" ")) { str = str.substring(0, str.length() - 1); } return str; } /** * Gets the current time formatted for HTTP headers */ protected String getDateHeader() { TimeZone tz = TimeZone.getTimeZone("GMT"); l4j.debug("TZ: " + tz); // Per the Java documentation, DateFormat objects are not thread safe. synchronized (HEADER_FORMAT) { HEADER_FORMAT.setTimeZone(tz); String dateHeader = HEADER_FORMAT.format(new Date(System.currentTimeMillis() - serverOffset)); l4j.debug("Date: " + dateHeader); return dateHeader; } } protected ObjectId getObjectId(String location) { Matcher m = OBJECTID_EXTRACTOR.matcher(location); if (m.find()) { String vid = m.group(1); l4j.debug("vId: " + vid); return new ObjectId(vid); } else { throw new EsuException("Could not find ObjectId in " + location); } } protected byte[] readStream(InputStream in, int contentLength) throws IOException { try { byte[] output; // If we know the content length, read it directly into a buffer. if (contentLength != -1) { output = new byte[contentLength]; int c = 0; while (c < contentLength) { int read = in.read(output, c, contentLength - c); if (read == -1) { // EOF! throw new EOFException( "EOF reading response at position " + c + " size " + (contentLength - c)); } c += read; } return output; } else { l4j.debug("Content length is unknown. Buffering output."); // Else, use a ByteArrayOutputStream to collect the response. byte[] buffer = new byte[4096]; ByteArrayOutputStream baos = new ByteArrayOutputStream(); int c = 0; while ((c = in.read(buffer)) != -1) { baos.write(buffer, 0, c); } baos.close(); l4j.debug("Buffered " + baos.size() + " response bytes"); return baos.toByteArray(); } } finally { if (in != null) { in.close(); } } } protected ServiceInformation parseServiceInformation(byte[] response, Map<String, List<String>> map) { // Use JDOM to parse the XML SAXBuilder sb = new SAXBuilder(); try { Document d = sb.build(new ByteArrayInputStream(response)); ServiceInformation si = new ServiceInformation(); // The ObjectID element is part of a namespace so we need to use // the namespace to identify the elements. Namespace esuNs = Namespace.getNamespace("http://www.emc.com/cos/"); Element ver = d.getRootElement().getChild("Version", esuNs); Element atmos = ver.getChild("Atmos", esuNs); si.setAtmosVersion(atmos.getTextNormalize()); // Check for UTF8 support for (String key : map.keySet()) { if ("x-emc-support-utf8".equalsIgnoreCase(key)) { for (String val : map.get(key)) { if ("true".equalsIgnoreCase(val)) { si.setUnicodeMetadataSupported(true); } } } if ("x-emc-features".equalsIgnoreCase(key)) { for (String val : map.get(key)) { String[] features = val.split(","); for (String feature : features) { si.addFeature(feature.trim()); } } } } return si; } catch (JDOMException e) { throw new EsuException("Error parsing response", e); } catch (IOException e) { throw new EsuException("Error reading response", e); } } /** * Converts an ObjectResult list to an Identifier list. */ private List<Identifier> filterIdList(List<ObjectResult> list) { List<Identifier> result = new ArrayList<Identifier>(list.size()); for (ObjectResult r : list) { result.add(r.getId()); } return result; } /** * Joins a list of Strings using a delimiter (similar to PERL, PHP, etc) * @param list the list of Strings * @param delimiter the string to join the list with * @return the joined String. */ protected String join(List<String> list, String delimiter) { boolean first = true; StringBuffer sb = new StringBuffer(); for (String s : list) { if (first) { first = false; } else { sb.append(delimiter); } sb.append(s); } return sb.toString(); } /** * @return the readChecksum */ public boolean isReadChecksum() { return readChecksum; } /** * Turns read checksum verification on or off. Note that * checksums are only returned from the server for erasure coded objects. * @param readChecksum the readChecksum to set */ public void setReadChecksum(boolean readChecksum) { this.readChecksum = readChecksum; } /** * Returns true if unicode metadata processing is enabled. */ public boolean isUnicodeEnabled() { return unicodeEnabled; } /** * Set to true to enable Unicode metadata processing. */ public void setUnicodeEnabled(boolean unicodeEnabled) { this.unicodeEnabled = unicodeEnabled; } /** * Gets the current server offset in milliseconds. This value can be used * to adjust for clock skew between the client and server. * @return the serverOffset */ public long getServerOffset() { return serverOffset; } /** * Sets the server offset in millesconds. This value can be used to * adjust for clock skew between the client and the server. * @param serverOffset the serverOffset to set */ public void setServerOffset(long serverOffset) { this.serverOffset = serverOffset; } /** * Makes a request to the server to get the value of the response Date * header. Compares this date with the local system time to calculate * the offset between the client and the server. You can pass this value * to the setServerOffset method to adjust for clock skew. * @return the offset between the client and server in milliseconds. If * the client is ahead of the server, this will be positive. If the server * is ahead of the client, it will be negative. */ public abstract long calculateServerOffset(); //---------- Features supported by the Atmos 2.0 REST API. ----------\\ @Override public ObjectId createObjectWithKey(String keyPool, String key, Acl acl, MetadataList metadata, byte[] data, long length, String mimeType) { return createObjectWithKeyFromSegment(keyPool, key, acl, metadata, new BufferSegment(data, 0, (int) length), mimeType); } @Override public ObjectId createObjectWithKey(String keyPool, String key, Acl acl, MetadataList metadata, byte[] data, long length, String mimeType, Checksum checksum) { return createObjectWithKeyFromSegment(keyPool, key, acl, metadata, new BufferSegment(data, 0, (int) length), mimeType, checksum); } @Override public ObjectId createObjectWithKeyFromSegment(String keyPool, String key, Acl acl, MetadataList metadata, BufferSegment data, String mimeType) { return createObjectWithKeyFromSegment(keyPool, key, acl, metadata, data, mimeType, null); } @Override public byte[] readObjectWithKey(String keyPool, String key, Extent extent, byte[] buffer) { return readObjectWithKey(keyPool, key, extent, buffer, null); } @Override public void updateObjectWithKey(String keyPool, String key, Acl acl, MetadataList metadata, Extent extent, byte[] data, String mimeType) { updateObjectWithKeyFromSegment(keyPool, key, acl, metadata, extent, new BufferSegment(data), mimeType); } @Override public void updateObjectWithKey(String keyPool, String key, Acl acl, MetadataList metadata, Extent extent, byte[] data, String mimeType, Checksum checksum) { updateObjectWithKeyFromSegment(keyPool, key, acl, metadata, extent, new BufferSegment(data), mimeType, checksum); } @Override public void updateObjectWithKeyFromSegment(String keyPool, String key, Acl acl, MetadataList metadata, Extent extent, BufferSegment data, String mimeType) { updateObjectWithKeyFromSegment(keyPool, key, acl, metadata, extent, data, mimeType, null); } }