Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.ibm.stocator.fs.cos; import java.io.EOFException; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InterruptedIOException; import java.nio.file.AccessDeniedException; import java.util.Date; import java.util.concurrent.ExecutionException; import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonServiceException; import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.ibm.stocator.fs.cos.exception.COSClientIOException; import com.ibm.stocator.fs.cos.exception.COSIOException; import com.ibm.stocator.fs.cos.exception.COSServiceIOException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static com.ibm.stocator.fs.cos.COSConstants.MULTIPART_MIN_SIZE; import static com.ibm.stocator.fs.cos.COSConstants.ENDPOINT_URL; /** * Utility methods for COS code. */ @InterfaceAudience.Private @InterfaceStability.Evolving public final class COSUtils { private static final Logger LOG = LoggerFactory.getLogger(COSUtils.class); static final String ENDPOINT_KEY = "Endpoint"; private COSUtils() { } /** * Translate an exception raised in an operation into an IOException. The * specific type of IOException depends on the class of * {@link AmazonClientException} passed in, and any status codes included in * the operation. That is: HTTP error codes are examined and can be used to * build a more specific response. * * @param operation operation * @param path path operated on (must not be null) * @param exception amazon exception raised * @return an IOE which wraps the caught exception */ public static IOException translateException(String operation, Path path, AmazonClientException exception) { return translateException(operation, path.toString(), exception); } /** * Translate an exception raised in an operation into an IOException. The * specific type of IOException depends on the class of * {@link AmazonClientException} passed in, and any status codes included in * the operation. That is: HTTP error codes are examined and can be used to * build a more specific response. * * @param operation operation * @param path path operated on (may be null) * @param exception amazon exception raised * @return an IOE which wraps the caught exception */ @SuppressWarnings("ThrowableInstanceNeverThrown") public static IOException translateException(String operation, String path, AmazonClientException exception) { String message = String.format("%s%s: %s", operation, path != null ? (" on " + path) : "", exception); if (!(exception instanceof AmazonServiceException)) { if (containsInterruptedException(exception)) { return (IOException) new InterruptedIOException(message).initCause(exception); } return new COSClientIOException(message, exception); } else { IOException ioe; AmazonServiceException ase = (AmazonServiceException) exception; // this exception is non-null if the service exception is an COS one AmazonS3Exception s3Exception = ase instanceof AmazonS3Exception ? (AmazonS3Exception) ase : null; int status = ase.getStatusCode(); switch (status) { case 301: if (s3Exception != null) { if (s3Exception.getAdditionalDetails() != null && s3Exception.getAdditionalDetails().containsKey(ENDPOINT_KEY)) { message = String.format( "Received permanent redirect response to " + "endpoint %s. This likely indicates that the COS endpoint " + "configured in %s does not match the region containing " + "the bucket.", s3Exception.getAdditionalDetails().get(ENDPOINT_KEY), ENDPOINT_URL); } ioe = new COSIOException(message, s3Exception); } else { ioe = new COSServiceIOException(message, ase); } break; // permissions case 401: case 403: ioe = new AccessDeniedException(path, null, message); ioe.initCause(ase); break; // the object isn't there case 404: case 410: ioe = new FileNotFoundException(message); ioe.initCause(ase); break; // out of range. This may happen if an object is overwritten with // a shorter one while it is being read. case 416: ioe = new EOFException(message); break; default: // no specific exit code. Choose an IOE subclass based on the class // of the caught exception ioe = s3Exception != null ? new COSIOException(message, s3Exception) : new COSServiceIOException(message, ase); break; } return ioe; } } /** * Extract an exception from a failed future, and convert to an IOE. * * @param operation operation which failed * @param path path operated on (may be null) * @param ee execution exception * @return an IOE which can be thrown */ public static IOException extractException(String operation, String path, ExecutionException ee) { IOException ioe; Throwable cause = ee.getCause(); if (cause instanceof AmazonClientException) { ioe = translateException(operation, path, (AmazonClientException) cause); } else if (cause instanceof IOException) { ioe = (IOException) cause; } else { ioe = new IOException(operation + " failed: " + cause, cause); } return ioe; } /** * Recurse down the exception loop looking for any inner details about an * interrupted exception. * * @param thrown exception thrown * @return true if down the execution chain the operation was an interrupt */ static boolean containsInterruptedException(Throwable thrown) { if (thrown == null) { return false; } if (thrown instanceof InterruptedException || thrown instanceof InterruptedIOException) { return true; } // tail recurse return containsInterruptedException(thrown.getCause()); } /** * Get a size property from the configuration: this property must be at least * equal to {@link COSConstants#MULTIPART_MIN_SIZE}. If it is too small, it is * rounded up to that minimum, and a warning printed. * * @param conf configuration * @param property property name * @param defVal default value * @return the value, guaranteed to be above the minimum size */ public static long getMultipartSizeProperty(Configuration conf, String property, long defVal) { long partSize = conf.getLongBytes(property, defVal); if (partSize < MULTIPART_MIN_SIZE) { LOG.warn("{} must be at least 5 MB; configured value is {}", property, partSize); partSize = MULTIPART_MIN_SIZE; } return partSize; } /** * Ensure that the long value is in the range of an integer. * * @param name property name for error messages * @param size original size * @return the size, guaranteed to be less than or equal to the max value of * an integer */ public static int ensureOutputParameterInRange(String name, long size) { if (size > Integer.MAX_VALUE) { LOG.warn("cos: {} capped to ~2.14GB" + " (maximum allowed size with current output mechanism)", name); return Integer.MAX_VALUE; } else { return (int) size; } } /** * Create a files status instance from a listing. * @param keyPath path to entry * @param summary summary from AWS * @param blockSize block size to declare * @return a status entry */ public static COSFileStatus createFileStatus(Path keyPath, S3ObjectSummary summary, long blockSize) { long size = summary.getSize(); return createFileStatus(keyPath, objectRepresentsDirectory(summary.getKey(), size), size, summary.getLastModified(), blockSize); } /* Date 'modified' is ignored when isDir is true. */ private static COSFileStatus createFileStatus(Path keyPath, boolean isDir, long size, Date modified, long blockSize) { if (isDir) { return new COSFileStatus(true, false, keyPath); } else { return new COSFileStatus(size, dateToLong(modified), keyPath, blockSize); } } public static boolean objectRepresentsDirectory(final String name, final long size) { return !name.isEmpty() && name.charAt(name.length() - 1) == '/' && size == 0L; } /** * Date to long conversion. * Handles null Dates that can be returned by AWS by returning 0 * @param date date from AWS query * @return timestamp of the object */ public static long dateToLong(final Date date) { if (date == null) { return 0L; } return date.getTime(); } public static String stringify(S3ObjectSummary summary) { StringBuilder builder = new StringBuilder(summary.getKey().length() + 100); builder.append(summary.getKey()).append(' '); builder.append("size=").append(summary.getSize()); return builder.toString(); } }