Java tutorial
/* * Copyright 2010 Cloud.com, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.cloud.bridge.service.controller.s3; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStream; import java.util.ArrayList; import java.util.Date; import java.util.Enumeration; import java.util.List; import java.util.UUID; import javax.activation.DataHandler; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.xml.namespace.QName; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.stream.XMLOutputFactory; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamWriter; import org.apache.axiom.om.OMAbstractFactory; import org.apache.axiom.om.OMFactory; import org.apache.axis2.databinding.utils.writer.MTOMAwareXMLSerializer; import org.apache.log4j.Logger; import org.apache.commons.fileupload.MultipartStream; import org.hibernate.LockMode; import com.amazon.s3.GetBucketAccessControlPolicyResponse; import com.amazon.s3.GetObjectAccessControlPolicyResponse; import com.cloud.bridge.model.SBucket; import com.cloud.bridge.model.SHost; import com.cloud.bridge.model.SObject; import com.cloud.bridge.model.SObjectItem; import com.cloud.bridge.persist.PersistContext; import com.cloud.bridge.persist.dao.SBucketDao; import com.cloud.bridge.persist.dao.SObjectDao; import com.cloud.bridge.service.S3BucketAdapter; import com.cloud.bridge.service.S3Constants; import com.cloud.bridge.service.S3RestServlet; import com.cloud.bridge.service.S3SoapServiceImpl; import com.cloud.bridge.service.ServiceProvider; import com.cloud.bridge.service.ServletAction; import com.cloud.bridge.service.UserContext; import com.cloud.bridge.service.core.ec2.EC2Volume; import com.cloud.bridge.service.core.s3.S3AccessControlPolicy; import com.cloud.bridge.service.core.s3.S3AuthParams; import com.cloud.bridge.service.core.s3.S3ConditionalHeaders; import com.cloud.bridge.service.core.s3.S3DeleteObjectRequest; import com.cloud.bridge.service.core.s3.S3Engine; import com.cloud.bridge.service.core.s3.S3GetBucketAccessControlPolicyRequest; import com.cloud.bridge.service.core.s3.S3GetObjectAccessControlPolicyRequest; import com.cloud.bridge.service.core.s3.S3GetObjectRequest; import com.cloud.bridge.service.core.s3.S3GetObjectResponse; import com.cloud.bridge.service.core.s3.S3MetaDataEntry; import com.cloud.bridge.service.core.s3.S3PutObjectInlineRequest; import com.cloud.bridge.service.core.s3.S3PutObjectInlineResponse; import com.cloud.bridge.service.core.s3.S3PutObjectRequest; import com.cloud.bridge.service.core.s3.S3Response; import com.cloud.bridge.service.core.s3.S3SetObjectAccessControlPolicyRequest; import com.cloud.bridge.util.Converter; import com.cloud.bridge.util.DateHelper; import com.cloud.bridge.util.HeaderParam; import com.cloud.bridge.util.ServletRequestDataSource; /** * @author Kelven Yang */ public class S3ObjectAction implements ServletAction { protected final static Logger logger = Logger.getLogger(S3ObjectAction.class); private DocumentBuilderFactory dbf = null; private OMFactory factory = OMAbstractFactory.getOMFactory(); private XMLOutputFactory xmlOutFactory = XMLOutputFactory.newInstance(); public S3ObjectAction() { dbf = DocumentBuilderFactory.newInstance(); dbf.setNamespaceAware(true); } public void execute(HttpServletRequest request, HttpServletResponse response) throws IOException { String method = request.getMethod(); String queryString = request.getQueryString(); response.addHeader("x-amz-request-id", UUID.randomUUID().toString()); if (method.equalsIgnoreCase("GET")) { if (queryString != null && queryString.length() > 0) { if (queryString.equalsIgnoreCase("acl")) executeGetObjectAcl(request, response); } else executeGetObject(request, response); } else if (method.equalsIgnoreCase("PUT")) { if (queryString != null && queryString.length() > 0) { if (queryString.equalsIgnoreCase("acl")) executePutObjectAcl(request, response); } else executePutObject(request, response); } else if (method.equalsIgnoreCase("DELETE")) { executeDeleteObject(request, response); } else if (method.equalsIgnoreCase("HEAD")) { executeHeadObject(request, response); } else if (method.equalsIgnoreCase("POST")) { executePostObject(request, response); } else throw new IllegalArgumentException("Unsupported method in REST request"); } private void executeGetObjectAcl(HttpServletRequest request, HttpServletResponse response) throws IOException { String bucketName = (String) request.getAttribute(S3Constants.BUCKET_ATTR_KEY); String key = (String) request.getAttribute(S3Constants.OBJECT_ATTR_KEY); S3GetObjectAccessControlPolicyRequest engineRequest = new S3GetObjectAccessControlPolicyRequest(); engineRequest.setBucketName(bucketName); engineRequest.setKey(key); S3AccessControlPolicy engineResponse = ServiceProvider.getInstance().getS3Engine() .handleRequest(engineRequest); // -> serialize using the apache's Axiom classes GetObjectAccessControlPolicyResponse onePolicy = S3SoapServiceImpl .toGetObjectAccessControlPolicyResponse(engineResponse); try { OutputStream os = response.getOutputStream(); response.setStatus(200); response.setContentType("text/xml; charset=UTF-8"); XMLStreamWriter xmlWriter = xmlOutFactory.createXMLStreamWriter(os); String documentStart = new String("<?xml version=\"1.0\" encoding=\"UTF-8\"?>"); os.write(documentStart.getBytes()); MTOMAwareXMLSerializer MTOMWriter = new MTOMAwareXMLSerializer(xmlWriter); onePolicy.serialize(new QName("http://s3.amazonaws.com/doc/2006-03-01/", "GetObjectAccessControlPolicyResponse", "ns1"), factory, MTOMWriter); xmlWriter.flush(); xmlWriter.close(); os.close(); } catch (XMLStreamException e) { throw new IOException(e.toString()); } } private void executePutObjectAcl(HttpServletRequest request, HttpServletResponse response) throws IOException { S3PutObjectRequest putRequest = null; // -> reuse the Access Control List parsing code that was added to support DIME String bucketName = (String) request.getAttribute(S3Constants.BUCKET_ATTR_KEY); String key = (String) request.getAttribute(S3Constants.OBJECT_ATTR_KEY); try { putRequest = S3RestServlet.toEnginePutObjectRequest(request.getInputStream()); } catch (Exception e) { throw new IOException(e.toString()); } // -> reuse the SOAP code to save the passed in ACLs S3SetObjectAccessControlPolicyRequest engineRequest = new S3SetObjectAccessControlPolicyRequest(); engineRequest.setBucketName(bucketName); engineRequest.setKey(key); engineRequest.setAcl(putRequest.getAcl()); S3Response engineResponse = ServiceProvider.getInstance().getS3Engine().handleRequest(engineRequest); response.setStatus(engineResponse.getResultCode()); } private void executeGetObject(HttpServletRequest request, HttpServletResponse response) throws IOException { String bucket = (String) request.getAttribute(S3Constants.BUCKET_ATTR_KEY); String key = (String) request.getAttribute(S3Constants.OBJECT_ATTR_KEY); String[] paramList = null; S3GetObjectRequest engineRequest = new S3GetObjectRequest(); engineRequest.setBucketName(bucket); engineRequest.setKey(key); engineRequest.setInlineData(true); engineRequest.setReturnData(true); //engineRequest.setReturnMetadata(true); engineRequest = setRequestByteRange(request, engineRequest); // -> is this a request for a specific version of the object? look for "versionId=" in the query string String queryString = request.getQueryString(); if (null != queryString) { paramList = queryString.split("[&=]"); if (null != paramList) engineRequest.setVersion(returnParameter(paramList, "versionId")); } S3GetObjectResponse engineResponse = ServiceProvider.getInstance().getS3Engine() .handleRequest(engineRequest); response.setStatus(engineResponse.getResultCode()); String deleteMarker = engineResponse.getDeleteMarker(); if (null != deleteMarker) { response.addHeader("x-amz-delete-marker", "true"); response.addHeader("x-amz-version-id", deleteMarker); } else { String version = engineResponse.getVersion(); if (null != version) response.addHeader("x-amz-version-id", version); } // -> was the get conditional? if (!conditionPassed(request, response, engineResponse.getLastModified().getTime(), engineResponse.getETag())) return; // -> is there data to return // -> from the Amazon REST documentation it appears that Meta data is only returned as part of a HEAD request //returnMetaData( engineResponse, response ); DataHandler dataHandler = engineResponse.getData(); if (dataHandler != null) { response.addHeader("ETag", engineResponse.getETag()); response.addHeader("Last-Modified", DateHelper.getDateDisplayString(DateHelper.GMT_TIMEZONE, engineResponse.getLastModified().getTime(), "E, d MMM yyyy HH:mm:ss z")); response.setContentLength((int) engineResponse.getContentLength()); S3RestServlet.writeResponse(response, dataHandler.getInputStream()); } } private void executePutObject(HttpServletRequest request, HttpServletResponse response) throws IOException { String continueHeader = request.getHeader("Expect"); if (continueHeader != null && continueHeader.equalsIgnoreCase("100-continue")) { S3RestServlet.writeResponse(response, "HTTP/1.1 100 Continue\r\n"); } String contentType = request.getHeader("Content-Type"); long contentLength = Converter.toLong(request.getHeader("Content-Length"), 0); String bucket = (String) request.getAttribute(S3Constants.BUCKET_ATTR_KEY); String key = (String) request.getAttribute(S3Constants.OBJECT_ATTR_KEY); S3PutObjectInlineRequest engineRequest = new S3PutObjectInlineRequest(); engineRequest.setBucketName(bucket); engineRequest.setKey(key); engineRequest.setContentLength(contentLength); engineRequest.setMetaEntries(extractMetaData(request)); engineRequest.setCannedAccess(request.getHeader("x-amz-acl")); DataHandler dataHandler = new DataHandler(new ServletRequestDataSource(request)); engineRequest.setData(dataHandler); S3PutObjectInlineResponse engineResponse = ServiceProvider.getInstance().getS3Engine() .handleRequest(engineRequest); response.setHeader("ETag", engineResponse.getETag()); String version = engineResponse.getVersion(); if (null != version) response.addHeader("x-amz-version-id", version); } /** * Once versioining is turned on then to delete an object requires specifying a version * parameter. A deletion marker is set once versioning is turned on in a bucket. */ private void executeDeleteObject(HttpServletRequest request, HttpServletResponse response) throws IOException { String bucket = (String) request.getAttribute(S3Constants.BUCKET_ATTR_KEY); String key = (String) request.getAttribute(S3Constants.OBJECT_ATTR_KEY); String[] paramList = null; S3DeleteObjectRequest engineRequest = new S3DeleteObjectRequest(); engineRequest.setBucketName(bucket); engineRequest.setKey(key); // -> is this a request for a specific version of the object? look for "versionId=" in the query string String queryString = request.getQueryString(); if (null != queryString) { paramList = queryString.split("[&=]"); if (null != paramList) engineRequest.setVersion(returnParameter(paramList, "versionId")); } S3Response engineResponse = ServiceProvider.getInstance().getS3Engine().handleRequest(engineRequest); response.setStatus(engineResponse.getResultCode()); String version = engineRequest.getVersion(); if (null != version) response.addHeader("x-amz-version-id", version); } private void executeHeadObject(HttpServletRequest request, HttpServletResponse response) throws IOException { String bucket = (String) request.getAttribute(S3Constants.BUCKET_ATTR_KEY); String key = (String) request.getAttribute(S3Constants.OBJECT_ATTR_KEY); String[] paramList = null; S3GetObjectRequest engineRequest = new S3GetObjectRequest(); engineRequest.setBucketName(bucket); engineRequest.setKey(key); engineRequest.setInlineData(true); // -> need to set so we get ETag etc returned engineRequest.setReturnData(true); engineRequest.setReturnMetadata(true); engineRequest = setRequestByteRange(request, engineRequest); // -> is this a request for a specific version of the object? look for "versionId=" in the query string String queryString = request.getQueryString(); if (null != queryString) { paramList = queryString.split("[&=]"); if (null != paramList) engineRequest.setVersion(returnParameter(paramList, "versionId")); } S3GetObjectResponse engineResponse = ServiceProvider.getInstance().getS3Engine() .handleRequest(engineRequest); response.setStatus(engineResponse.getResultCode()); String deleteMarker = engineResponse.getDeleteMarker(); if (null != deleteMarker) { response.addHeader("x-amz-delete-marker", "true"); response.addHeader("x-amz-version-id", deleteMarker); } else { String version = engineResponse.getVersion(); if (null != version) response.addHeader("x-amz-version-id", version); } // -> was the head request conditional? if (!conditionPassed(request, response, engineResponse.getLastModified().getTime(), engineResponse.getETag())) return; // -> for a head request we return everything except the data returnMetaData(engineResponse, response); DataHandler dataHandler = engineResponse.getData(); if (dataHandler != null) { response.addHeader("ETag", engineResponse.getETag()); response.addHeader("Last-Modified", DateHelper.getDateDisplayString(DateHelper.GMT_TIMEZONE, engineResponse.getLastModified().getTime(), "E, d MMM yyyy HH:mm:ss z")); response.setContentLength((int) engineResponse.getContentLength()); } } // There is a problem with POST since the 'Signature' and 'AccessKey' parameters are not // determined until we hit this function (i.e., they are encoded in the body of the message // they are not HTTP request headers). All the values we used to get in the request headers // are not encoded in the request body. // public void executePostObject(HttpServletRequest request, HttpServletResponse response) throws IOException { String bucket = (String) request.getAttribute(S3Constants.BUCKET_ATTR_KEY); String contentType = request.getHeader("Content-Type"); int boundaryIndex = contentType.indexOf("boundary="); String boundary = "--" + (contentType.substring(boundaryIndex + 9)); String lastBoundary = boundary + "--"; InputStreamReader isr = new InputStreamReader(request.getInputStream()); BufferedReader br = new BufferedReader(isr); StringBuffer temp = new StringBuffer(); String oneLine = null; String name = null; String value = null; String metaName = null; // -> after stripped off the x-amz-meta- boolean isMetaTag = false; int countMeta = 0; int state = 0; // [A] First parse all the parts out of the POST request and message body // -> bucket name is still encoded in a Host header S3AuthParams params = new S3AuthParams(); List<S3MetaDataEntry> metaSet = new ArrayList<S3MetaDataEntry>(); S3PutObjectInlineRequest engineRequest = new S3PutObjectInlineRequest(); engineRequest.setBucketName(bucket); // -> the last body part contains the content that is used to write the S3 object, all // other body parts are header values while (null != (oneLine = br.readLine())) { if (oneLine.startsWith(lastBoundary)) { // -> this is the data of the object to put if (0 < temp.length()) { value = temp.toString(); temp.setLength(0); engineRequest.setContentLength(value.length()); engineRequest.setDataAsString(value); } break; } else if (oneLine.startsWith(boundary)) { // -> this is the header data if (0 < temp.length()) { value = temp.toString().trim(); temp.setLength(0); //System.out.println( "param: " + name + " = " + value ); if (name.equalsIgnoreCase("key")) { engineRequest.setKey(value); } else if (name.equalsIgnoreCase("x-amz-acl")) { engineRequest.setCannedAccess(value); } else if (isMetaTag) { S3MetaDataEntry oneMeta = new S3MetaDataEntry(); oneMeta.setName(metaName); oneMeta.setValue(value); metaSet.add(oneMeta); countMeta++; metaName = null; } // -> build up the headers so we can do authentication on this POST HeaderParam oneHeader = new HeaderParam(); oneHeader.setName(name); oneHeader.setValue(value); params.addHeader(oneHeader); } state = 1; } else if (1 == state && 0 == oneLine.length()) { // -> data of a body part starts here state = 2; } else if (1 == state) { // -> the name of the 'name-value' pair is encoded in the Content-Disposition header if (oneLine.startsWith("Content-Disposition: form-data;")) { isMetaTag = false; int nameOffset = oneLine.indexOf("name="); if (-1 != nameOffset) { name = oneLine.substring(nameOffset + 5); if (name.startsWith("\"")) name = name.substring(1); if (name.endsWith("\"")) name = name.substring(0, name.length() - 1); name = name.trim(); if (name.startsWith("x-amz-meta-")) { metaName = name.substring(11); isMetaTag = true; } } } } else if (2 == state) { // -> the body parts data may take up multiple lines //System.out.println( oneLine.length() + " body data: " + oneLine ); temp.append(oneLine); } // else System.out.println( oneLine.length() + " preamble: " + oneLine ); } // [B] Authenticate the POST request after we have all the headers try { S3RestServlet.authenticateRequest(request, params); } catch (Exception e) { throw new IOException(e.toString()); } // [C] Perform the request if (0 < countMeta) engineRequest.setMetaEntries(metaSet.toArray(new S3MetaDataEntry[0])); S3PutObjectInlineResponse engineResponse = ServiceProvider.getInstance().getS3Engine() .handleRequest(engineRequest); response.setHeader("ETag", engineResponse.getETag()); String version = engineResponse.getVersion(); if (null != version) response.addHeader("x-amz-version-id", version); } /** * Support the "Range: bytes=0-399" header with just one byte range. * @param request * @param engineRequest * @return */ private S3GetObjectRequest setRequestByteRange(HttpServletRequest request, S3GetObjectRequest engineRequest) { String temp = request.getHeader("Range"); if (null == temp) return engineRequest; int offset = temp.indexOf("="); if (-1 != offset) { String range = temp.substring(offset + 1); String[] parts = range.split("-"); if (2 >= parts.length) { // -> the end byte is inclusive engineRequest.setByteRangeStart(Long.parseLong(parts[0])); engineRequest.setByteRangeEnd(Long.parseLong(parts[1]) + 1); } } return engineRequest; } private S3ConditionalHeaders conditionalRequest(HttpServletRequest request) { S3ConditionalHeaders headers = new S3ConditionalHeaders(); headers.setModifiedSince(request.getHeader("If-Modified-Since")); headers.setUnModifiedSince(request.getHeader("If-Unmodified-Since")); headers.setMatch(request.getHeader("If-Match")); headers.setNoneMatch(request.getHeader("If-None-Match")); return headers; } private boolean conditionPassed(HttpServletRequest request, HttpServletResponse response, Date lastModified, String ETag) { S3ConditionalHeaders ifCond = conditionalRequest(request); if (0 > ifCond.ifModifiedSince(lastModified)) { response.setStatus(304); return false; } if (0 > ifCond.ifUnmodifiedSince(lastModified)) { response.setStatus(412); return false; } if (0 > ifCond.ifMatchEtag(ETag)) { response.setStatus(412); return false; } if (0 > ifCond.ifNoneMatchEtag(ETag)) { response.setStatus(412); return false; } return true; } /** * Return the saved object's meta data back to the client as HTTP "x-amz-meta-" headers. * This function is constructing an HTTP header and these headers have a defined syntax * as defined in rfc2616. Any characters that could cause an invalid HTTP header will * prevent that meta data from being returned via the REST call (as is defined in the Amazon * spec). These characters can be defined if using the SOAP API as well as the REST API. * * @param engineResponse * @param response */ private void returnMetaData(S3GetObjectResponse engineResponse, HttpServletResponse response) { boolean ignoreMeta = false; int ignoredCount = 0; S3MetaDataEntry[] metaSet = engineResponse.getMetaEntries(); for (int i = 0; null != metaSet && i < metaSet.length; i++) { String name = metaSet[i].getName(); String value = metaSet[i].getValue(); byte[] nameBytes = name.getBytes(); ignoreMeta = false; // -> cannot have control characters (octets 0 - 31) and DEL (127), in an HTTP header for (int j = 0; j < name.length(); j++) { if ((0 <= nameBytes[j] && 31 >= nameBytes[j]) || 127 == nameBytes[j]) { ignoreMeta = true; break; } } // -> cannot have HTTP separators in an HTTP header if (-1 != name.indexOf('(') || -1 != name.indexOf(')') || -1 != name.indexOf('@') || -1 != name.indexOf('<') || -1 != name.indexOf('>') || -1 != name.indexOf('\"') || -1 != name.indexOf('[') || -1 != name.indexOf(']') || -1 != name.indexOf('=') || -1 != name.indexOf(',') || -1 != name.indexOf(';') || -1 != name.indexOf(':') || -1 != name.indexOf('\\') || -1 != name.indexOf('/') || -1 != name.indexOf(' ') || -1 != name.indexOf('{') || -1 != name.indexOf('}') || -1 != name.indexOf('?') || -1 != name.indexOf('\t')) ignoreMeta = true; if (ignoreMeta) ignoredCount++; else response.addHeader("x-amz-meta-" + name, value); } if (0 < ignoredCount) response.addHeader("x-amz-missing-meta", new String("" + ignoredCount)); } /** * Extract the name and value of all meta data so it can be written with the * object that is being 'PUT'. * * @param request * @return */ private S3MetaDataEntry[] extractMetaData(HttpServletRequest request) { List<S3MetaDataEntry> metaSet = new ArrayList<S3MetaDataEntry>(); int count = 0; Enumeration headers = request.getHeaderNames(); while (headers.hasMoreElements()) { String key = (String) headers.nextElement(); if (key.startsWith("x-amz-meta-")) { String name = key.substring(11); String value = request.getHeader(key); if (null != value) { S3MetaDataEntry oneMeta = new S3MetaDataEntry(); oneMeta.setName(name); oneMeta.setValue(value); metaSet.add(oneMeta); count++; } } } if (0 < count) return metaSet.toArray(new S3MetaDataEntry[0]); else return null; } /** * @param paramList - name - value pairs with name at odd indexes * @param find - name string to return first found * @return the value matching the found name */ private String returnParameter(String[] paramList, String find) { int i = 0; if (paramList == null) return null; while (i + 2 <= paramList.length) { if (paramList[i].equalsIgnoreCase(find)) return paramList[i + 1]; i += 2; } return null; } }