Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.fs.azure; import static org.apache.hadoop.fs.azure.NativeAzureFileSystem.PATH_DELIMITER; import java.io.BufferedInputStream; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.UnsupportedEncodingException; import java.net.URI; import java.net.URISyntaxException; import java.net.URLDecoder; import java.net.URLEncoder; import java.security.InvalidKeyException; import java.util.ArrayList; import java.util.Calendar; import java.util.Collections; import java.util.Date; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedList; import java.util.Locale; import java.util.Map; import java.util.Set; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.azure.StorageInterface.CloudBlobContainerWrapper; import org.apache.hadoop.fs.azure.StorageInterface.CloudBlobDirectoryWrapper; import org.apache.hadoop.fs.azure.StorageInterface.CloudBlobWrapper; import org.apache.hadoop.fs.azure.StorageInterface.CloudBlockBlobWrapper; import org.apache.hadoop.fs.azure.StorageInterface.CloudPageBlobWrapper; import org.apache.hadoop.fs.azure.StorageInterfaceImpl.CloudPageBlobWrapperImpl; import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation; import org.apache.hadoop.fs.azure.metrics.BandwidthGaugeUpdater; import org.apache.hadoop.fs.azure.metrics.ErrorMetricUpdater; import org.apache.hadoop.fs.azure.metrics.ResponseReceivedMetricUpdater; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; import org.mortbay.util.ajax.JSON; import com.google.common.annotations.VisibleForTesting; import com.microsoft.azure.storage.CloudStorageAccount; import com.microsoft.azure.storage.OperationContext; import com.microsoft.azure.storage.RetryExponentialRetry; import com.microsoft.azure.storage.RetryNoRetry; import com.microsoft.azure.storage.StorageCredentials; import com.microsoft.azure.storage.StorageCredentialsAccountAndKey; import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature; import com.microsoft.azure.storage.StorageErrorCode; import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.blob.BlobListingDetails; import com.microsoft.azure.storage.blob.BlobProperties; import com.microsoft.azure.storage.blob.BlobRequestOptions; import com.microsoft.azure.storage.blob.CloudBlob; import com.microsoft.azure.storage.blob.CopyStatus; import com.microsoft.azure.storage.blob.DeleteSnapshotsOption; import com.microsoft.azure.storage.blob.ListBlobItem; import com.microsoft.azure.storage.core.Utility; /** * Core implementation of Windows Azure Filesystem for Hadoop. * Provides the bridging logic between Hadoop's abstract filesystem and Azure Storage * */ @InterfaceAudience.Private @VisibleForTesting public class AzureNativeFileSystemStore implements NativeFileSystemStore { /** * Configuration knob on whether we do block-level MD5 validation on * upload/download. */ static final String KEY_CHECK_BLOCK_MD5 = "fs.azure.check.block.md5"; /** * Configuration knob on whether we store blob-level MD5 on upload. */ static final String KEY_STORE_BLOB_MD5 = "fs.azure.store.blob.md5"; static final String DEFAULT_STORAGE_EMULATOR_ACCOUNT_NAME = "storageemulator"; static final String STORAGE_EMULATOR_ACCOUNT_NAME_PROPERTY_NAME = "fs.azure.storage.emulator.account.name"; public static final Log LOG = LogFactory.getLog(AzureNativeFileSystemStore.class); private StorageInterface storageInteractionLayer; private CloudBlobDirectoryWrapper rootDirectory; private CloudBlobContainerWrapper container; // Constants local to this class. // private static final String KEY_ACCOUNT_KEYPROVIDER_PREFIX = "fs.azure.account.keyprovider."; private static final String KEY_ACCOUNT_SAS_PREFIX = "fs.azure.sas."; // note: this value is not present in core-default.xml as our real default is // computed as min(2*cpu,8) private static final String KEY_CONCURRENT_CONNECTION_VALUE_OUT = "fs.azure.concurrentRequestCount.out"; private static final String KEY_STREAM_MIN_READ_SIZE = "fs.azure.read.request.size"; private static final String KEY_STORAGE_CONNECTION_TIMEOUT = "fs.azure.storage.timeout"; private static final String KEY_WRITE_BLOCK_SIZE = "fs.azure.write.request.size"; // Property controlling whether to allow reads on blob which are concurrently // appended out-of-band. private static final String KEY_READ_TOLERATE_CONCURRENT_APPEND = "fs.azure.io.read.tolerate.concurrent.append"; // Configurable throttling parameter properties. These properties are located // in the core-site.xml configuration file. private static final String KEY_MIN_BACKOFF_INTERVAL = "fs.azure.io.retry.min.backoff.interval"; private static final String KEY_MAX_BACKOFF_INTERVAL = "fs.azure.io.retry.max.backoff.interval"; private static final String KEY_BACKOFF_INTERVAL = "fs.azure.io.retry.backoff.interval"; private static final String KEY_MAX_IO_RETRIES = "fs.azure.io.retry.max.retries"; private static final String KEY_COPYBLOB_MIN_BACKOFF_INTERVAL = "fs.azure.io.copyblob.retry.min.backoff.interval"; private static final String KEY_COPYBLOB_MAX_BACKOFF_INTERVAL = "fs.azure.io.copyblob.retry.max.backoff.interval"; private static final String KEY_COPYBLOB_BACKOFF_INTERVAL = "fs.azure.io.copyblob.retry.backoff.interval"; private static final String KEY_COPYBLOB_MAX_IO_RETRIES = "fs.azure.io.copyblob.retry.max.retries"; private static final String KEY_SELF_THROTTLE_ENABLE = "fs.azure.selfthrottling.enable"; private static final String KEY_SELF_THROTTLE_READ_FACTOR = "fs.azure.selfthrottling.read.factor"; private static final String KEY_SELF_THROTTLE_WRITE_FACTOR = "fs.azure.selfthrottling.write.factor"; private static final String KEY_ENABLE_STORAGE_CLIENT_LOGGING = "fs.azure.storage.client.logging"; private static final String PERMISSION_METADATA_KEY = "hdi_permission"; private static final String OLD_PERMISSION_METADATA_KEY = "asv_permission"; private static final String IS_FOLDER_METADATA_KEY = "hdi_isfolder"; private static final String OLD_IS_FOLDER_METADATA_KEY = "asv_isfolder"; static final String VERSION_METADATA_KEY = "hdi_version"; static final String OLD_VERSION_METADATA_KEY = "asv_version"; static final String FIRST_WASB_VERSION = "2013-01-01"; static final String CURRENT_WASB_VERSION = "2013-09-01"; static final String LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY = "hdi_tmpupload"; static final String OLD_LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY = "asv_tmpupload"; /** * Configuration key to indicate the set of directories in WASB where we * should store files as page blobs instead of block blobs. * * Entries should be plain directory names (i.e. not URIs) with no leading or * trailing slashes. Delimit the entries with commas. */ public static final String KEY_PAGE_BLOB_DIRECTORIES = "fs.azure.page.blob.dir"; /** * The set of directories where we should store files as page blobs. */ private Set<String> pageBlobDirs; /** * Configuration key to indicate the set of directories in WASB where * we should do atomic folder rename synchronized with createNonRecursive. */ public static final String KEY_ATOMIC_RENAME_DIRECTORIES = "fs.azure.atomic.rename.dir"; /** * The set of directories where we should apply atomic folder rename * synchronized with createNonRecursive. */ private Set<String> atomicRenameDirs; private static final String HTTP_SCHEME = "http"; private static final String HTTPS_SCHEME = "https"; private static final String WASB_AUTHORITY_DELIMITER = "@"; private static final String AZURE_ROOT_CONTAINER = "$root"; private static final int DEFAULT_CONCURRENT_WRITES = 8; // Concurrent reads reads of data written out of band are disable by default. // private static final boolean DEFAULT_READ_TOLERATE_CONCURRENT_APPEND = false; // Default block sizes public static final int DEFAULT_DOWNLOAD_BLOCK_SIZE = 4 * 1024 * 1024; public static final int DEFAULT_UPLOAD_BLOCK_SIZE = 4 * 1024 * 1024; // Retry parameter defaults. // private static final int DEFAULT_MIN_BACKOFF_INTERVAL = 1 * 1000; // 1s private static final int DEFAULT_MAX_BACKOFF_INTERVAL = 30 * 1000; // 30s private static final int DEFAULT_BACKOFF_INTERVAL = 1 * 1000; // 1s private static final int DEFAULT_MAX_RETRY_ATTEMPTS = 15; private static final int DEFAULT_COPYBLOB_MIN_BACKOFF_INTERVAL = 3 * 1000; private static final int DEFAULT_COPYBLOB_MAX_BACKOFF_INTERVAL = 90 * 1000; private static final int DEFAULT_COPYBLOB_BACKOFF_INTERVAL = 30 * 1000; private static final int DEFAULT_COPYBLOB_MAX_RETRY_ATTEMPTS = 15; // Self-throttling defaults. Allowed range = (0,1.0] // Value of 1.0 means no self-throttling. // Value of x means process data at factor x of unrestricted rate private static final boolean DEFAULT_SELF_THROTTLE_ENABLE = true; private static final float DEFAULT_SELF_THROTTLE_READ_FACTOR = 1.0f; private static final float DEFAULT_SELF_THROTTLE_WRITE_FACTOR = 1.0f; private static final int STORAGE_CONNECTION_TIMEOUT_DEFAULT = 90; /** * MEMBER VARIABLES */ private URI sessionUri; private Configuration sessionConfiguration; private int concurrentWrites = DEFAULT_CONCURRENT_WRITES; private boolean isAnonymousCredentials = false; // Set to true if we are connecting using shared access signatures. private boolean connectingUsingSAS = false; private AzureFileSystemInstrumentation instrumentation; private BandwidthGaugeUpdater bandwidthGaugeUpdater; private final static JSON PERMISSION_JSON_SERIALIZER = createPermissionJsonSerializer(); private boolean suppressRetryPolicy = false; private boolean canCreateOrModifyContainer = false; private ContainerState currentKnownContainerState = ContainerState.Unknown; private final Object containerStateLock = new Object(); private boolean tolerateOobAppends = DEFAULT_READ_TOLERATE_CONCURRENT_APPEND; private int downloadBlockSizeBytes = DEFAULT_DOWNLOAD_BLOCK_SIZE; private int uploadBlockSizeBytes = DEFAULT_UPLOAD_BLOCK_SIZE; // Bandwidth throttling exponential back-off parameters // private int minBackoff; // the minimum back-off interval (ms) between retries. private int maxBackoff; // the maximum back-off interval (ms) between retries. private int deltaBackoff; // the back-off interval (ms) between retries. private int maxRetries; // the maximum number of retry attempts. // Self-throttling parameters private boolean selfThrottlingEnabled; private float selfThrottlingReadFactor; private float selfThrottlingWriteFactor; private TestHookOperationContext testHookOperationContext = null; // Set if we're running against a storage emulator.. private boolean isStorageEmulator = false; /** * A test hook interface that can modify the operation context we use for * Azure Storage operations, e.g. to inject errors. */ @VisibleForTesting interface TestHookOperationContext { OperationContext modifyOperationContext(OperationContext original); } /** * Suppress the default retry policy for the Storage, useful in unit tests to * test negative cases without waiting forever. */ @VisibleForTesting void suppressRetryPolicy() { suppressRetryPolicy = true; } /** * Add a test hook to modify the operation context we use for Azure Storage * operations. * * @param testHook * The test hook, or null to unset previous hooks. */ @VisibleForTesting void addTestHookToOperationContext(TestHookOperationContext testHook) { this.testHookOperationContext = testHook; } /** * If we're asked by unit tests to not retry, set the retry policy factory in * the client accordingly. */ private void suppressRetryPolicyInClientIfNeeded() { if (suppressRetryPolicy) { storageInteractionLayer.setRetryPolicyFactory(new RetryNoRetry()); } } /** * Creates a JSON serializer that can serialize a PermissionStatus object into * the JSON string we want in the blob metadata. * * @return The JSON serializer. */ private static JSON createPermissionJsonSerializer() { JSON serializer = new JSON(); serializer.addConvertor(PermissionStatus.class, new PermissionStatusJsonSerializer()); return serializer; } /** * A converter for PermissionStatus to/from JSON as we want it in the blob * metadata. */ private static class PermissionStatusJsonSerializer implements JSON.Convertor { private static final String OWNER_TAG = "owner"; private static final String GROUP_TAG = "group"; private static final String PERMISSIONS_TAG = "permissions"; @Override public void toJSON(Object obj, JSON.Output out) { PermissionStatus permissionStatus = (PermissionStatus) obj; // Don't store group as null, just store it as empty string // (which is FileStatus behavior). String group = permissionStatus.getGroupName() == null ? "" : permissionStatus.getGroupName(); out.add(OWNER_TAG, permissionStatus.getUserName()); out.add(GROUP_TAG, group); out.add(PERMISSIONS_TAG, permissionStatus.getPermission().toString()); } @Override public Object fromJSON(@SuppressWarnings("rawtypes") Map object) { return PermissionStatusJsonSerializer.fromJSONMap(object); } @SuppressWarnings("rawtypes") public static PermissionStatus fromJSONString(String jsonString) { // The JSON class can only find out about an object's class (and call me) // if we store the class name in the JSON string. Since I don't want to // do that (it's an implementation detail), I just deserialize as a // the default Map (JSON's default behavior) and parse that. return fromJSONMap((Map) PERMISSION_JSON_SERIALIZER.fromJSON(jsonString)); } private static PermissionStatus fromJSONMap(@SuppressWarnings("rawtypes") Map object) { return new PermissionStatus((String) object.get(OWNER_TAG), (String) object.get(GROUP_TAG), // The initial - below is the Unix file type, // which FsPermission needs there but ignores. FsPermission.valueOf("-" + (String) object.get(PERMISSIONS_TAG))); } } @VisibleForTesting void setAzureStorageInteractionLayer(StorageInterface storageInteractionLayer) { this.storageInteractionLayer = storageInteractionLayer; } @VisibleForTesting public BandwidthGaugeUpdater getBandwidthGaugeUpdater() { return bandwidthGaugeUpdater; } /** * Check if concurrent reads and writes on the same blob are allowed. * * @return true if concurrent reads and OOB writes has been configured, false * otherwise. */ private boolean isConcurrentOOBAppendAllowed() { return tolerateOobAppends; } /** * Method for the URI and configuration object necessary to create a storage * session with an Azure session. It parses the scheme to ensure it matches * the storage protocol supported by this file system. * * @param uri - URI for target storage blob. * @param conf - reference to configuration object. * @param instrumentation - the metrics source that will keep track of operations here. * * @throws IllegalArgumentException if URI or job object is null, or invalid scheme. */ @Override public void initialize(URI uri, Configuration conf, AzureFileSystemInstrumentation instrumentation) throws IllegalArgumentException, AzureException, IOException { if (null == instrumentation) { throw new IllegalArgumentException("Null instrumentation"); } this.instrumentation = instrumentation; if (null == this.storageInteractionLayer) { this.storageInteractionLayer = new StorageInterfaceImpl(); } // Check that URI exists. // if (null == uri) { throw new IllegalArgumentException("Cannot initialize WASB file system, URI is null"); } // Check that configuration object is non-null. // if (null == conf) { throw new IllegalArgumentException("Cannot initialize WASB file system, conf is null"); } if (!conf.getBoolean(NativeAzureFileSystem.SKIP_AZURE_METRICS_PROPERTY_NAME, false)) { //If not skip azure metrics, create bandwidthGaugeUpdater this.bandwidthGaugeUpdater = new BandwidthGaugeUpdater(instrumentation); } // Incoming parameters validated. Capture the URI and the job configuration // object. // sessionUri = uri; sessionConfiguration = conf; // Start an Azure storage session. // createAzureStorageSession(); // Extract the directories that should contain page blobs pageBlobDirs = getDirectorySet(KEY_PAGE_BLOB_DIRECTORIES); LOG.debug("Page blob directories: " + setToString(pageBlobDirs)); // Extract directories that should have atomic rename applied. atomicRenameDirs = getDirectorySet(KEY_ATOMIC_RENAME_DIRECTORIES); String hbaseRoot; try { // Add to this the hbase root directory, or /hbase is that is not set. hbaseRoot = verifyAndConvertToStandardFormat(sessionConfiguration.get("hbase.rootdir", "hbase")); atomicRenameDirs.add(hbaseRoot); } catch (URISyntaxException e) { LOG.warn("Unable to initialize HBase root as an atomic rename directory."); } LOG.debug("Atomic rename directories: " + setToString(atomicRenameDirs)); } /** * Helper to format a string for log output from Set<String> */ private String setToString(Set<String> set) { StringBuilder sb = new StringBuilder(); int i = 1; for (String s : set) { sb.append("/" + s); if (i != set.size()) { sb.append(", "); } i++; } return sb.toString(); } /** * Method to extract the account name from an Azure URI. * * @param uri * -- WASB blob URI * @returns accountName -- the account name for the URI. * @throws URISyntaxException * if the URI does not have an authority it is badly formed. */ private String getAccountFromAuthority(URI uri) throws URISyntaxException { // Check to make sure that the authority is valid for the URI. // String authority = uri.getRawAuthority(); if (null == authority) { // Badly formed or illegal URI. // throw new URISyntaxException(uri.toString(), "Expected URI with a valid authority"); } // Check if authority container the delimiter separating the account name from the // the container. // if (!authority.contains(WASB_AUTHORITY_DELIMITER)) { return authority; } // Split off the container name and the authority. // String[] authorityParts = authority.split(WASB_AUTHORITY_DELIMITER, 2); // Because the string contains an '@' delimiter, a container must be // specified. // if (authorityParts.length < 2 || "".equals(authorityParts[0])) { // Badly formed WASB authority since there is no container. // final String errMsg = String.format("URI '%s' has a malformed WASB authority, expected container name. " + "Authority takes the form wasb://[<container name>@]<account name>", uri.toString()); throw new IllegalArgumentException(errMsg); } // Return with the account name. It is possible that this name is NULL. // return authorityParts[1]; } /** * Method to extract the container name from an Azure URI. * * @param uri * -- WASB blob URI * @returns containerName -- the container name for the URI. May be null. * @throws URISyntaxException * if the uri does not have an authority it is badly formed. */ private String getContainerFromAuthority(URI uri) throws URISyntaxException { // Check to make sure that the authority is valid for the URI. // String authority = uri.getRawAuthority(); if (null == authority) { // Badly formed or illegal URI. // throw new URISyntaxException(uri.toString(), "Expected URI with a valid authority"); } // The URI has a valid authority. Extract the container name. It is the // second component of the WASB URI authority. if (!authority.contains(WASB_AUTHORITY_DELIMITER)) { // The authority does not have a container name. Use the default container by // setting the container name to the default Azure root container. // return AZURE_ROOT_CONTAINER; } // Split off the container name and the authority. String[] authorityParts = authority.split(WASB_AUTHORITY_DELIMITER, 2); // Because the string contains an '@' delimiter, a container must be // specified. if (authorityParts.length < 2 || "".equals(authorityParts[0])) { // Badly formed WASB authority since there is no container. final String errMsg = String.format("URI '%s' has a malformed WASB authority, expected container name." + "Authority takes the form wasb://[<container name>@]<account name>", uri.toString()); throw new IllegalArgumentException(errMsg); } // Set the container name from the first entry for the split parts of the // authority. return authorityParts[0]; } /** * Get the appropriate return the appropriate scheme for communicating with * Azure depending on whether wasb or wasbs is specified in the target URI. * * return scheme - HTTPS or HTTP as appropriate. */ private String getHTTPScheme() { String sessionScheme = sessionUri.getScheme(); // Check if we're on a secure URI scheme: wasbs or the legacy asvs scheme. if (sessionScheme != null && (sessionScheme.equalsIgnoreCase("asvs") || sessionScheme.equalsIgnoreCase("wasbs"))) { return HTTPS_SCHEME; } else { // At this point the scheme should be either null or asv or wasb. // Intentionally I'm not going to validate it though since I don't feel // it's this method's job to ensure a valid URI scheme for this file // system. return HTTP_SCHEME; } } /** * Set the configuration parameters for this client storage session with * Azure. * * @throws AzureException */ private void configureAzureStorageSession() throws AzureException { // Assertion: Target session URI already should have been captured. if (sessionUri == null) { throw new AssertionError("Expected a non-null session URI when configuring storage session"); } // Assertion: A client session already should have been established with // Azure. if (storageInteractionLayer == null) { throw new AssertionError(String.format("Cannot configure storage session for URI '%s' " + "if storage session has not been established.", sessionUri.toString())); } // Determine whether or not reads are allowed concurrent with OOB writes. tolerateOobAppends = sessionConfiguration.getBoolean(KEY_READ_TOLERATE_CONCURRENT_APPEND, DEFAULT_READ_TOLERATE_CONCURRENT_APPEND); // Retrieve configuration for the minimum stream read and write block size. // this.downloadBlockSizeBytes = sessionConfiguration.getInt(KEY_STREAM_MIN_READ_SIZE, DEFAULT_DOWNLOAD_BLOCK_SIZE); this.uploadBlockSizeBytes = sessionConfiguration.getInt(KEY_WRITE_BLOCK_SIZE, DEFAULT_UPLOAD_BLOCK_SIZE); // The job may want to specify a timeout to use when engaging the // storage service. The default is currently 90 seconds. It may // be necessary to increase this value for long latencies in larger // jobs. If the timeout specified is greater than zero seconds use // it, otherwise use the default service client timeout. int storageConnectionTimeout = sessionConfiguration.getInt(KEY_STORAGE_CONNECTION_TIMEOUT, 0); if (0 < storageConnectionTimeout) { storageInteractionLayer.setTimeoutInMs(storageConnectionTimeout * 1000); } // Set the concurrency values equal to the that specified in the // configuration file. If it does not exist, set it to the default // value calculated as double the number of CPU cores on the client // machine. The concurrency value is minimum of double the cores and // the read/write property. int cpuCores = 2 * Runtime.getRuntime().availableProcessors(); concurrentWrites = sessionConfiguration.getInt(KEY_CONCURRENT_CONNECTION_VALUE_OUT, Math.min(cpuCores, DEFAULT_CONCURRENT_WRITES)); // Set up the exponential retry policy. // minBackoff = sessionConfiguration.getInt(KEY_MIN_BACKOFF_INTERVAL, DEFAULT_MIN_BACKOFF_INTERVAL); maxBackoff = sessionConfiguration.getInt(KEY_MAX_BACKOFF_INTERVAL, DEFAULT_MAX_BACKOFF_INTERVAL); deltaBackoff = sessionConfiguration.getInt(KEY_BACKOFF_INTERVAL, DEFAULT_BACKOFF_INTERVAL); maxRetries = sessionConfiguration.getInt(KEY_MAX_IO_RETRIES, DEFAULT_MAX_RETRY_ATTEMPTS); storageInteractionLayer .setRetryPolicyFactory(new RetryExponentialRetry(minBackoff, deltaBackoff, maxBackoff, maxRetries)); // read the self-throttling config. selfThrottlingEnabled = sessionConfiguration.getBoolean(KEY_SELF_THROTTLE_ENABLE, DEFAULT_SELF_THROTTLE_ENABLE); selfThrottlingReadFactor = sessionConfiguration.getFloat(KEY_SELF_THROTTLE_READ_FACTOR, DEFAULT_SELF_THROTTLE_READ_FACTOR); selfThrottlingWriteFactor = sessionConfiguration.getFloat(KEY_SELF_THROTTLE_WRITE_FACTOR, DEFAULT_SELF_THROTTLE_WRITE_FACTOR); OperationContext.setLoggingEnabledByDefault( sessionConfiguration.getBoolean(KEY_ENABLE_STORAGE_CLIENT_LOGGING, false)); if (LOG.isDebugEnabled()) { LOG.debug(String.format("AzureNativeFileSystemStore init. Settings=%d,%b,%d,{%d,%d,%d,%d},{%b,%f,%f}", concurrentWrites, tolerateOobAppends, ((storageConnectionTimeout > 0) ? storageConnectionTimeout : STORAGE_CONNECTION_TIMEOUT_DEFAULT), minBackoff, deltaBackoff, maxBackoff, maxRetries, selfThrottlingEnabled, selfThrottlingReadFactor, selfThrottlingWriteFactor)); } } /** * Connect to Azure storage using anonymous credentials. * * @param uri * - URI to target blob (R/O access to public blob) * * @throws StorageException * raised on errors communicating with Azure storage. * @throws IOException * raised on errors performing I/O or setting up the session. * @throws URISyntaxException * raised on creating mal-formed URI's. */ private void connectUsingAnonymousCredentials(final URI uri) throws StorageException, IOException, URISyntaxException { // Use an HTTP scheme since the URI specifies a publicly accessible // container. Explicitly create a storage URI corresponding to the URI // parameter for use in creating the service client. String accountName = getAccountFromAuthority(uri); URI storageUri = new URI(getHTTPScheme() + ":" + PATH_DELIMITER + PATH_DELIMITER + accountName); // Create the service client with anonymous credentials. String containerName = getContainerFromAuthority(uri); storageInteractionLayer.createBlobClient(storageUri); suppressRetryPolicyInClientIfNeeded(); // Capture the container reference. container = storageInteractionLayer.getContainerReference(containerName); rootDirectory = container.getDirectoryReference(""); // Check for container existence, and our ability to access it. try { if (!container.exists(getInstrumentedContext())) { throw new AzureException("Container " + containerName + " in account " + accountName + " not found, and we can't create " + " it using anoynomous credentials."); } } catch (StorageException ex) { throw new AzureException("Unable to access container " + containerName + " in account " + accountName + " using anonymous credentials, and no credentials found for them " + " in the configuration.", ex); } // Accessing the storage server unauthenticated using // anonymous credentials. isAnonymousCredentials = true; // Configure Azure storage session. configureAzureStorageSession(); } private void connectUsingCredentials(String accountName, StorageCredentials credentials, String containerName) throws URISyntaxException, StorageException, AzureException { URI blobEndPoint; if (isStorageEmulatorAccount(accountName)) { isStorageEmulator = true; CloudStorageAccount account = CloudStorageAccount.getDevelopmentStorageAccount(); storageInteractionLayer.createBlobClient(account); } else { blobEndPoint = new URI(getHTTPScheme() + "://" + accountName); storageInteractionLayer.createBlobClient(blobEndPoint, credentials); } suppressRetryPolicyInClientIfNeeded(); // Capture the container reference for debugging purposes. container = storageInteractionLayer.getContainerReference(containerName); rootDirectory = container.getDirectoryReference(""); // Can only create container if using account key credentials canCreateOrModifyContainer = credentials instanceof StorageCredentialsAccountAndKey; // Configure Azure storage session. configureAzureStorageSession(); } /** * Connect to Azure storage using account key credentials. */ private void connectUsingConnectionStringCredentials(final String accountName, final String containerName, final String accountKey) throws InvalidKeyException, StorageException, IOException, URISyntaxException { // If the account name is "acc.blob.core.windows.net", then the // rawAccountName is just "acc" String rawAccountName = accountName.split("\\.")[0]; StorageCredentials credentials = new StorageCredentialsAccountAndKey(rawAccountName, accountKey); connectUsingCredentials(accountName, credentials, containerName); } /** * Connect to Azure storage using shared access signature credentials. */ private void connectUsingSASCredentials(final String accountName, final String containerName, final String sas) throws InvalidKeyException, StorageException, IOException, URISyntaxException { StorageCredentials credentials = new StorageCredentialsSharedAccessSignature(sas); connectingUsingSAS = true; connectUsingCredentials(accountName, credentials, containerName); } private boolean isStorageEmulatorAccount(final String accountName) { return accountName.equalsIgnoreCase(sessionConfiguration.get(STORAGE_EMULATOR_ACCOUNT_NAME_PROPERTY_NAME, DEFAULT_STORAGE_EMULATOR_ACCOUNT_NAME)); } @VisibleForTesting public static String getAccountKeyFromConfiguration(String accountName, Configuration conf) throws KeyProviderException { String key = null; String keyProviderClass = conf.get(KEY_ACCOUNT_KEYPROVIDER_PREFIX + accountName); KeyProvider keyProvider = null; if (keyProviderClass == null) { // No key provider was provided so use the provided key as is. keyProvider = new SimpleKeyProvider(); } else { // create an instance of the key provider class and verify it // implements KeyProvider Object keyProviderObject = null; try { Class<?> clazz = conf.getClassByName(keyProviderClass); keyProviderObject = clazz.newInstance(); } catch (Exception e) { throw new KeyProviderException("Unable to load key provider class.", e); } if (!(keyProviderObject instanceof KeyProvider)) { throw new KeyProviderException( keyProviderClass + " specified in config is not a valid KeyProvider class."); } keyProvider = (KeyProvider) keyProviderObject; } key = keyProvider.getStorageAccountKey(accountName, conf); return key; } /** * Establish a session with Azure blob storage based on the target URI. The * method determines whether or not the URI target contains an explicit * account or an implicit default cluster-wide account. * * @throws AzureException * @throws IOException */ private void createAzureStorageSession() throws AzureException, IOException { // Make sure this object was properly initialized with references to // the sessionUri and sessionConfiguration. if (null == sessionUri || null == sessionConfiguration) { throw new AzureException("Filesystem object not initialized properly." + "Unable to start session with Azure Storage server."); } // File system object initialized, attempt to establish a session // with the Azure storage service for the target URI string. try { // Inspect the URI authority to determine the account and use the account // to start an Azure blob client session using an account key for the // the account or anonymously. // For all URI's do the following checks in order: // 1. Validate that <account> can be used with the current Hadoop // cluster by checking it exists in the list of configured accounts // for the cluster. // 2. Look up the AccountKey in the list of configured accounts for the // cluster. // 3. If there is no AccountKey, assume anonymous public blob access // when accessing the blob. // // If the URI does not specify a container use the default root container // under the account name. // Assertion: Container name on the session Uri should be non-null. if (getContainerFromAuthority(sessionUri) == null) { throw new AssertionError( String.format("Non-null container expected from session URI: %s.", sessionUri.toString())); } // Get the account name. String accountName = getAccountFromAuthority(sessionUri); if (null == accountName) { // Account name is not specified as part of the URI. Throw indicating // an invalid account name. final String errMsg = String.format( "Cannot load WASB file system account name not" + " specified in URI: %s.", sessionUri.toString()); throw new AzureException(errMsg); } instrumentation.setAccountName(accountName); String containerName = getContainerFromAuthority(sessionUri); instrumentation.setContainerName(containerName); // Check whether this is a storage emulator account. if (isStorageEmulatorAccount(accountName)) { // It is an emulator account, connect to it with no credentials. connectUsingCredentials(accountName, null, containerName); return; } // Check whether we have a shared access signature for that container. String propertyValue = sessionConfiguration .get(KEY_ACCOUNT_SAS_PREFIX + containerName + "." + accountName); if (propertyValue != null) { // SAS was found. Connect using that. connectUsingSASCredentials(accountName, containerName, propertyValue); return; } // Check whether the account is configured with an account key. propertyValue = getAccountKeyFromConfiguration(accountName, sessionConfiguration); if (propertyValue != null) { // Account key was found. // Create the Azure storage session using the account key and container. connectUsingConnectionStringCredentials(getAccountFromAuthority(sessionUri), getContainerFromAuthority(sessionUri), propertyValue); // Return to caller return; } // The account access is not configured for this cluster. Try anonymous // access. connectUsingAnonymousCredentials(sessionUri); } catch (Exception e) { // Caught exception while attempting to initialize the Azure File // System store, re-throw the exception. throw new AzureException(e); } } private enum ContainerState { /** * We haven't checked the container state yet. */ Unknown, /** * We checked and the container doesn't exist. */ DoesntExist, /** * The container exists and doesn't have an WASB version stamp on it. */ ExistsNoVersion, /** * The container exists and has an unsupported WASB version stamped on it. */ ExistsAtWrongVersion, /** * The container exists and has the proper WASB version stamped on it. */ ExistsAtRightVersion } private enum ContainerAccessType { /** * We're accessing the container for a pure read operation, e.g. read a * file. */ PureRead, /** * We're accessing the container purely to write something, e.g. write a * file. */ PureWrite, /** * We're accessing the container to read something then write, e.g. rename a * file. */ ReadThenWrite } /** * Trims a suffix/prefix from the given string. For example if * s is given as "/xy" and toTrim is "/", this method returns "xy" */ private static String trim(String s, String toTrim) { return StringUtils.removeEnd(StringUtils.removeStart(s, toTrim), toTrim); } /** * Checks if the given rawDir belongs to this account/container, and * if so returns the canonicalized path for it. Otherwise return null. */ private String verifyAndConvertToStandardFormat(String rawDir) throws URISyntaxException { URI asUri = new URI(rawDir); if (asUri.getAuthority() == null || asUri.getAuthority().toLowerCase(Locale.ENGLISH) .equalsIgnoreCase(sessionUri.getAuthority().toLowerCase(Locale.ENGLISH))) { // Applies to me. return trim(asUri.getPath(), "/"); } else { // Doen't apply to me. return null; } } /** * Take a comma-separated list of directories from a configuration variable * and transform it to a set of directories. */ private Set<String> getDirectorySet(final String configVar) throws AzureException { String[] rawDirs = sessionConfiguration.getStrings(configVar, new String[0]); Set<String> directorySet = new HashSet<String>(); for (String currentDir : rawDirs) { String myDir; try { myDir = verifyAndConvertToStandardFormat(currentDir); } catch (URISyntaxException ex) { throw new AzureException(String.format( "The directory %s specified in the configuration entry %s is not" + " a valid URI.", currentDir, configVar)); } if (myDir != null) { directorySet.add(myDir); } } return directorySet; } /** * Checks if the given key in Azure Storage should be stored as a page * blob instead of block blob. */ public boolean isPageBlobKey(String key) { return isKeyForDirectorySet(key, pageBlobDirs); } /** * Checks if the given key in Azure storage should have synchronized * atomic folder rename createNonRecursive implemented. */ @Override public boolean isAtomicRenameKey(String key) { return isKeyForDirectorySet(key, atomicRenameDirs); } public boolean isKeyForDirectorySet(String key, Set<String> dirSet) { String defaultFS = FileSystem.getDefaultUri(sessionConfiguration).toString(); for (String dir : dirSet) { if (dir.isEmpty() || key.startsWith(dir + "/")) { return true; } // Allow for blob directories with paths relative to the default file // system. // try { URI uriPageBlobDir = new URI(dir); if (null == uriPageBlobDir.getAuthority()) { // Concatenate the default file system prefix with the relative // page blob directory path. // if (key.startsWith(trim(defaultFS, "/") + "/" + dir + "/")) { return true; } } } catch (URISyntaxException e) { LOG.info(String.format("URI syntax error creating URI for %s", dir)); } } return false; } /** * This should be called from any method that does any modifications to the * underlying container: it makes sure to put the WASB current version in the * container's metadata if it's not already there. */ private ContainerState checkContainer(ContainerAccessType accessType) throws StorageException, AzureException { synchronized (containerStateLock) { if (isOkContainerState(accessType)) { return currentKnownContainerState; } if (currentKnownContainerState == ContainerState.ExistsAtWrongVersion) { String containerVersion = retrieveVersionAttribute(container); throw wrongVersionException(containerVersion); } // This means I didn't check it before or it didn't exist or // we need to stamp the version. Since things may have changed by // other machines since then, do the check again and don't depend // on past information. // Sanity check: we don't expect this at this point. if (currentKnownContainerState == ContainerState.ExistsAtRightVersion) { throw new AssertionError("Unexpected state: " + currentKnownContainerState); } // Download the attributes - doubles as an existence check with just // one service call try { container.downloadAttributes(getInstrumentedContext()); currentKnownContainerState = ContainerState.Unknown; } catch (StorageException ex) { if (ex.getErrorCode().equals(StorageErrorCode.RESOURCE_NOT_FOUND.toString())) { currentKnownContainerState = ContainerState.DoesntExist; } else { throw ex; } } if (currentKnownContainerState == ContainerState.DoesntExist) { // If the container doesn't exist and we intend to write to it, // create it now. if (needToCreateContainer(accessType)) { storeVersionAttribute(container); container.create(getInstrumentedContext()); currentKnownContainerState = ContainerState.ExistsAtRightVersion; } } else { // The container exists, check the version. String containerVersion = retrieveVersionAttribute(container); if (containerVersion != null) { if (containerVersion.equals(FIRST_WASB_VERSION)) { // It's the version from when WASB was called ASV, just // fix the version attribute if needed and proceed. // We should be good otherwise. if (needToStampVersion(accessType)) { storeVersionAttribute(container); container.uploadMetadata(getInstrumentedContext()); } } else if (!containerVersion.equals(CURRENT_WASB_VERSION)) { // Don't know this version - throw. currentKnownContainerState = ContainerState.ExistsAtWrongVersion; throw wrongVersionException(containerVersion); } else { // It's our correct version. currentKnownContainerState = ContainerState.ExistsAtRightVersion; } } else { // No version info exists. currentKnownContainerState = ContainerState.ExistsNoVersion; if (needToStampVersion(accessType)) { // Need to stamp the version storeVersionAttribute(container); container.uploadMetadata(getInstrumentedContext()); currentKnownContainerState = ContainerState.ExistsAtRightVersion; } } } return currentKnownContainerState; } } private AzureException wrongVersionException(String containerVersion) { return new AzureException("The container " + container.getName() + " is at an unsupported version: " + containerVersion + ". Current supported version: " + FIRST_WASB_VERSION); } private boolean needToStampVersion(ContainerAccessType accessType) { // We need to stamp the version on the container any time we write to // it and we have the correct credentials to be able to write container // metadata. return accessType != ContainerAccessType.PureRead && canCreateOrModifyContainer; } private static boolean needToCreateContainer(ContainerAccessType accessType) { // We need to pro-actively create the container (if it doesn't exist) if // we're doing a pure write. No need to create it for pure read or read- // then-write access. return accessType == ContainerAccessType.PureWrite; } // Determines whether we have to pull the container information again // or we can work based off what we already have. private boolean isOkContainerState(ContainerAccessType accessType) { switch (currentKnownContainerState) { case Unknown: // When using SAS, we can't discover container attributes // so just live with Unknown state and fail later if it // doesn't exist. return connectingUsingSAS; case DoesntExist: return false; // the container could have been created case ExistsAtRightVersion: return true; // fine to optimize case ExistsAtWrongVersion: return false; case ExistsNoVersion: // If there's no version, it's OK if we don't need to stamp the version // or we can't anyway even if we wanted to. return !needToStampVersion(accessType); default: throw new AssertionError("Unknown access type: " + accessType); } } private boolean getUseTransactionalContentMD5() { return sessionConfiguration.getBoolean(KEY_CHECK_BLOCK_MD5, true); } private BlobRequestOptions getUploadOptions() { BlobRequestOptions options = new BlobRequestOptions(); options.setStoreBlobContentMD5(sessionConfiguration.getBoolean(KEY_STORE_BLOB_MD5, false)); options.setUseTransactionalContentMD5(getUseTransactionalContentMD5()); options.setConcurrentRequestCount(concurrentWrites); options.setRetryPolicyFactory(new RetryExponentialRetry(minBackoff, deltaBackoff, maxBackoff, maxRetries)); return options; } private BlobRequestOptions getDownloadOptions() { BlobRequestOptions options = new BlobRequestOptions(); options.setRetryPolicyFactory(new RetryExponentialRetry(minBackoff, deltaBackoff, maxBackoff, maxRetries)); options.setUseTransactionalContentMD5(getUseTransactionalContentMD5()); return options; } @Override public DataOutputStream storefile(String key, PermissionStatus permissionStatus) throws AzureException { try { // Check if a session exists, if not create a session with the // Azure storage server. if (null == storageInteractionLayer) { final String errMsg = String.format("Storage session expected for URI '%s' but does not exist.", sessionUri); throw new AzureException(errMsg); } // Check if there is an authenticated account associated with the // file this instance of the WASB file system. If not the file system // has not been authenticated and all access is anonymous. if (!isAuthenticatedAccess()) { // Preemptively raise an exception indicating no uploads are // allowed to anonymous accounts. throw new AzureException( new IOException("Uploads to public accounts using anonymous " + "access is prohibited.")); } checkContainer(ContainerAccessType.PureWrite); /** * Note: Windows Azure Blob Storage does not allow the creation of arbitrary directory * paths under the default $root directory. This is by design to eliminate * ambiguity in specifying a implicit blob address. A blob in the $root conatiner * cannot include a / in its name and must be careful not to include a trailing * '/' when referencing blobs in the $root container. * A '/; in the $root container permits ambiguous blob names as in the following * example involving two containers $root and mycontainer: * http://myaccount.blob.core.windows.net/$root * http://myaccount.blob.core.windows.net/mycontainer * If the URL "mycontainer/somefile.txt were allowed in $root then the URL: * http://myaccount.blob.core.windows.net/mycontainer/myblob.txt * could mean either: * (1) container=mycontainer; blob=myblob.txt * (2) container=$root; blob=mycontainer/myblob.txt * * To avoid this type of ambiguity the Azure blob storage prevents * arbitrary path under $root. For a simple and more consistent user * experience it was decided to eliminate the opportunity for creating * such paths by making the $root container read-only under WASB. */ // Check that no attempt is made to write to blobs on default // $root containers. if (AZURE_ROOT_CONTAINER.equals(getContainerFromAuthority(sessionUri))) { // Azure containers are restricted to non-root containers. final String errMsg = String.format( "Writes to '%s' container for URI '%s' are prohibited, " + "only updates on non-root containers permitted.", AZURE_ROOT_CONTAINER, sessionUri.toString()); throw new AzureException(errMsg); } // Get the blob reference from the store's container and // return it. CloudBlobWrapper blob = getBlobReference(key); storePermissionStatus(blob, permissionStatus); // Create the output stream for the Azure blob. // OutputStream outputStream = openOutputStream(blob); DataOutputStream dataOutStream = new SyncableDataOutputStream(outputStream); return dataOutStream; } catch (Exception e) { // Caught exception while attempting to open the blob output stream. // Re-throw as an Azure storage exception. throw new AzureException(e); } } /** * Opens a new output stream to the given blob (page or block blob) * to populate it from scratch with data. */ private OutputStream openOutputStream(final CloudBlobWrapper blob) throws StorageException { if (blob instanceof CloudPageBlobWrapperImpl) { return new PageBlobOutputStream((CloudPageBlobWrapper) blob, getInstrumentedContext(), sessionConfiguration); } else { // Handle both ClouldBlockBlobWrapperImpl and (only for the test code path) // MockCloudBlockBlobWrapper. return ((CloudBlockBlobWrapper) blob).openOutputStream(getUploadOptions(), getInstrumentedContext()); } } /** * Opens a new input stream for the given blob (page or block blob) * to read its data. */ private InputStream openInputStream(CloudBlobWrapper blob) throws StorageException, IOException { if (blob instanceof CloudBlockBlobWrapper) { return blob.openInputStream(getDownloadOptions(), getInstrumentedContext(isConcurrentOOBAppendAllowed())); } else { return new PageBlobInputStream((CloudPageBlobWrapper) blob, getInstrumentedContext(isConcurrentOOBAppendAllowed())); } } /** * Default permission to use when no permission metadata is found. * * @return The default permission to use. */ private static PermissionStatus defaultPermissionNoBlobMetadata() { return new PermissionStatus("", "", FsPermission.getDefault()); } private static void storeMetadataAttribute(CloudBlobWrapper blob, String key, String value) { HashMap<String, String> metadata = blob.getMetadata(); if (null == metadata) { metadata = new HashMap<String, String>(); } metadata.put(key, value); blob.setMetadata(metadata); } private static String getMetadataAttribute(CloudBlobWrapper blob, String... keyAlternatives) { HashMap<String, String> metadata = blob.getMetadata(); if (null == metadata) { return null; } for (String key : keyAlternatives) { if (metadata.containsKey(key)) { return metadata.get(key); } } return null; } private static void removeMetadataAttribute(CloudBlobWrapper blob, String key) { HashMap<String, String> metadata = blob.getMetadata(); if (metadata != null) { metadata.remove(key); blob.setMetadata(metadata); } } private static void storePermissionStatus(CloudBlobWrapper blob, PermissionStatus permissionStatus) { storeMetadataAttribute(blob, PERMISSION_METADATA_KEY, PERMISSION_JSON_SERIALIZER.toJSON(permissionStatus)); // Remove the old metadata key if present removeMetadataAttribute(blob, OLD_PERMISSION_METADATA_KEY); } private PermissionStatus getPermissionStatus(CloudBlobWrapper blob) { String permissionMetadataValue = getMetadataAttribute(blob, PERMISSION_METADATA_KEY, OLD_PERMISSION_METADATA_KEY); if (permissionMetadataValue != null) { return PermissionStatusJsonSerializer.fromJSONString(permissionMetadataValue); } else { return defaultPermissionNoBlobMetadata(); } } private static void storeFolderAttribute(CloudBlobWrapper blob) { storeMetadataAttribute(blob, IS_FOLDER_METADATA_KEY, "true"); // Remove the old metadata key if present removeMetadataAttribute(blob, OLD_IS_FOLDER_METADATA_KEY); } private static void storeLinkAttribute(CloudBlobWrapper blob, String linkTarget) throws UnsupportedEncodingException { // We have to URL encode the link attribute as the link URI could // have URI special characters which unless encoded will result // in 403 errors from the server. This is due to metadata properties // being sent in the HTTP header of the request which is in turn used // on the server side to authorize the request. String encodedLinkTarget = null; if (linkTarget != null) { encodedLinkTarget = URLEncoder.encode(linkTarget, "UTF-8"); } storeMetadataAttribute(blob, LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY, encodedLinkTarget); // Remove the old metadata key if present removeMetadataAttribute(blob, OLD_LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY); } private static String getLinkAttributeValue(CloudBlobWrapper blob) throws UnsupportedEncodingException { String encodedLinkTarget = getMetadataAttribute(blob, LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY, OLD_LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY); String linkTarget = null; if (encodedLinkTarget != null) { linkTarget = URLDecoder.decode(encodedLinkTarget, "UTF-8"); } return linkTarget; } private static boolean retrieveFolderAttribute(CloudBlobWrapper blob) { HashMap<String, String> metadata = blob.getMetadata(); return null != metadata && (metadata.containsKey(IS_FOLDER_METADATA_KEY) || metadata.containsKey(OLD_IS_FOLDER_METADATA_KEY)); } private static void storeVersionAttribute(CloudBlobContainerWrapper container) { HashMap<String, String> metadata = container.getMetadata(); if (null == metadata) { metadata = new HashMap<String, String>(); } metadata.put(VERSION_METADATA_KEY, CURRENT_WASB_VERSION); if (metadata.containsKey(OLD_VERSION_METADATA_KEY)) { metadata.remove(OLD_VERSION_METADATA_KEY); } container.setMetadata(metadata); } private static String retrieveVersionAttribute(CloudBlobContainerWrapper container) { HashMap<String, String> metadata = container.getMetadata(); if (metadata == null) { return null; } else if (metadata.containsKey(VERSION_METADATA_KEY)) { return metadata.get(VERSION_METADATA_KEY); } else if (metadata.containsKey(OLD_VERSION_METADATA_KEY)) { return metadata.get(OLD_VERSION_METADATA_KEY); } else { return null; } } @Override public void storeEmptyFolder(String key, PermissionStatus permissionStatus) throws AzureException { if (null == storageInteractionLayer) { final String errMsg = String.format("Storage session expected for URI '%s' but does not exist.", sessionUri); throw new AssertionError(errMsg); } // Check if there is an authenticated account associated with the file // this instance of the WASB file system. If not the file system has not // been authenticated and all access is anonymous. if (!isAuthenticatedAccess()) { // Preemptively raise an exception indicating no uploads are // allowed to anonymous accounts. throw new AzureException("Uploads to to public accounts using anonymous access is prohibited."); } try { checkContainer(ContainerAccessType.PureWrite); CloudBlobWrapper blob = getBlobReference(key); storePermissionStatus(blob, permissionStatus); storeFolderAttribute(blob); openOutputStream(blob).close(); } catch (Exception e) { // Caught exception while attempting upload. Re-throw as an Azure // storage exception. throw new AzureException(e); } } /** * Stores an empty blob that's linking to the temporary file where're we're * uploading the initial data. */ @Override public void storeEmptyLinkFile(String key, String tempBlobKey, PermissionStatus permissionStatus) throws AzureException { if (null == storageInteractionLayer) { final String errMsg = String.format("Storage session expected for URI '%s' but does not exist.", sessionUri); throw new AssertionError(errMsg); } // Check if there is an authenticated account associated with the file // this instance of the WASB file system. If not the file system has not // been authenticated and all access is anonymous. if (!isAuthenticatedAccess()) { // Preemptively raise an exception indicating no uploads are // allowed to anonymous accounts. throw new AzureException("Uploads to to public accounts using anonymous access is prohibited."); } try { checkContainer(ContainerAccessType.PureWrite); CloudBlobWrapper blob = getBlobReference(key); storePermissionStatus(blob, permissionStatus); storeLinkAttribute(blob, tempBlobKey); openOutputStream(blob).close(); } catch (Exception e) { // Caught exception while attempting upload. Re-throw as an Azure // storage exception. throw new AzureException(e); } } /** * If the blob with the given key exists and has a link in its metadata to a * temporary file (see storeEmptyLinkFile), this method returns the key to * that temporary file. Otherwise, returns null. */ @Override public String getLinkInFileMetadata(String key) throws AzureException { if (null == storageInteractionLayer) { final String errMsg = String.format("Storage session expected for URI '%s' but does not exist.", sessionUri); throw new AssertionError(errMsg); } try { checkContainer(ContainerAccessType.PureRead); CloudBlobWrapper blob = getBlobReference(key); blob.downloadAttributes(getInstrumentedContext()); return getLinkAttributeValue(blob); } catch (Exception e) { // Caught exception while attempting download. Re-throw as an Azure // storage exception. throw new AzureException(e); } } /** * Private method to check for authenticated access. * * @ returns boolean -- true if access is credentialed and authenticated and * false otherwise. */ private boolean isAuthenticatedAccess() throws AzureException { if (isAnonymousCredentials) { // Access to this storage account is unauthenticated. return false; } // Access is authenticated. return true; } /** * This private method uses the root directory or the original container to * list blobs under the directory or container depending on whether the * original file system object was constructed with a short- or long-form URI. * If the root directory is non-null the URI in the file constructor was in * the long form. * * @param includeMetadata * if set, the listed items will have their metadata populated * already. * * @returns blobItems : iterable collection of blob items. * @throws URISyntaxException * */ private Iterable<ListBlobItem> listRootBlobs(boolean includeMetadata) throws StorageException, URISyntaxException { return rootDirectory.listBlobs(null, false, includeMetadata ? EnumSet.of(BlobListingDetails.METADATA) : EnumSet.noneOf(BlobListingDetails.class), null, getInstrumentedContext()); } /** * This private method uses the root directory or the original container to * list blobs under the directory or container given a specified prefix for * the directory depending on whether the original file system object was * constructed with a short- or long-form URI. If the root directory is * non-null the URI in the file constructor was in the long form. * * @param aPrefix * : string name representing the prefix of containing blobs. * @param includeMetadata * if set, the listed items will have their metadata populated * already. * * @returns blobItems : iterable collection of blob items. * @throws URISyntaxException * */ private Iterable<ListBlobItem> listRootBlobs(String aPrefix, boolean includeMetadata) throws StorageException, URISyntaxException { Iterable<ListBlobItem> list = rootDirectory.listBlobs(aPrefix, false, includeMetadata ? EnumSet.of(BlobListingDetails.METADATA) : EnumSet.noneOf(BlobListingDetails.class), null, getInstrumentedContext()); return list; } /** * This private method uses the root directory or the original container to * list blobs under the directory or container given a specified prefix for * the directory depending on whether the original file system object was * constructed with a short- or long-form URI. It also uses the specified flat * or hierarchical option, listing details options, request options, and * operation context. * * @param aPrefix * string name representing the prefix of containing blobs. * @param useFlatBlobListing * - the list is flat if true, or hierarchical otherwise. * @param listingDetails * - determine whether snapshots, metadata, committed/uncommitted * data * @param options * - object specifying additional options for the request. null = * default options * @param opContext * - context of the current operation * @returns blobItems : iterable collection of blob items. * @throws URISyntaxException * */ private Iterable<ListBlobItem> listRootBlobs(String aPrefix, boolean useFlatBlobListing, EnumSet<BlobListingDetails> listingDetails, BlobRequestOptions options, OperationContext opContext) throws StorageException, URISyntaxException { CloudBlobDirectoryWrapper directory = this.container.getDirectoryReference(aPrefix); return directory.listBlobs(null, useFlatBlobListing, listingDetails, options, opContext); } /** * This private method uses the root directory or the original container to * get the block blob reference depending on whether the original file system * object was constructed with a short- or long-form URI. If the root * directory is non-null the URI in the file constructor was in the long form. * * @param aKey * : a key used to query Azure for the block blob. * @returns blob : a reference to the Azure block blob corresponding to the * key. * @throws URISyntaxException * */ private CloudBlobWrapper getBlobReference(String aKey) throws StorageException, URISyntaxException { CloudBlobWrapper blob = null; if (isPageBlobKey(aKey)) { blob = this.container.getPageBlobReference(aKey); } else { blob = this.container.getBlockBlobReference(aKey); blob.setStreamMinimumReadSizeInBytes(downloadBlockSizeBytes); blob.setWriteBlockSizeInBytes(uploadBlockSizeBytes); } return blob; } /** * This private method normalizes the key by stripping the container name from * the path and returns a path relative to the root directory of the * container. * * @param keyUri * - adjust this key to a path relative to the root directory * * @returns normKey */ private String normalizeKey(URI keyUri) { String normKey; // Strip the container name from the path and return the path // relative to the root directory of the container. int parts = isStorageEmulator ? 4 : 3; normKey = keyUri.getPath().split("/", parts)[(parts - 1)]; // Return the fixed key. return normKey; } /** * This private method normalizes the key by stripping the container name from * the path and returns a path relative to the root directory of the * container. * * @param blob * - adjust the key to this blob to a path relative to the root * directory * * @returns normKey */ private String normalizeKey(CloudBlobWrapper blob) { return normalizeKey(blob.getUri()); } /** * This private method normalizes the key by stripping the container name from * the path and returns a path relative to the root directory of the * container. * * @param directory * - adjust the key to this directory to a path relative to the root * directory * * @returns normKey */ private String normalizeKey(CloudBlobDirectoryWrapper directory) { String dirKey = normalizeKey(directory.getUri()); // Strip the last delimiter if (dirKey.endsWith(PATH_DELIMITER)) { dirKey = dirKey.substring(0, dirKey.length() - 1); } return dirKey; } /** * Default method to creates a new OperationContext for the Azure Storage * operation that has listeners hooked to it that will update the metrics for * this file system. This method does not bind to receive send request * callbacks by default. * * @return The OperationContext object to use. */ private OperationContext getInstrumentedContext() { // Default is to not bind to receive send callback events. return getInstrumentedContext(false); } /** * Creates a new OperationContext for the Azure Storage operation that has * listeners hooked to it that will update the metrics for this file system. * * @param bindConcurrentOOBIo * - bind to intercept send request call backs to handle OOB I/O. * * @return The OperationContext object to use. */ private OperationContext getInstrumentedContext(boolean bindConcurrentOOBIo) { OperationContext operationContext = new OperationContext(); if (selfThrottlingEnabled) { SelfThrottlingIntercept.hook(operationContext, selfThrottlingReadFactor, selfThrottlingWriteFactor); } if (bandwidthGaugeUpdater != null) { //bandwidthGaugeUpdater is null when we config to skip azure metrics ResponseReceivedMetricUpdater.hook(operationContext, instrumentation, bandwidthGaugeUpdater); } // Bind operation context to receive send request callbacks on this operation. // If reads concurrent to OOB writes are allowed, the interception will reset // the conditional header on all Azure blob storage read requests. if (bindConcurrentOOBIo) { SendRequestIntercept.bind(storageInteractionLayer.getCredentials(), operationContext, true); } if (testHookOperationContext != null) { operationContext = testHookOperationContext.modifyOperationContext(operationContext); } ErrorMetricUpdater.hook(operationContext, instrumentation); // Return the operation context. return operationContext; } @Override public FileMetadata retrieveMetadata(String key) throws IOException { // Attempts to check status may occur before opening any streams so first, // check if a session exists, if not create a session with the Azure storage // server. if (null == storageInteractionLayer) { final String errMsg = String.format("Storage session expected for URI '%s' but does not exist.", sessionUri); throw new AssertionError(errMsg); } if (LOG.isDebugEnabled()) { LOG.debug("Retrieving metadata for " + key); } try { if (checkContainer(ContainerAccessType.PureRead) == ContainerState.DoesntExist) { // The container doesn't exist, so spare some service calls and just // return null now. return null; } // Handle the degenerate cases where the key does not exist or the // key is a container. if (key.equals("/")) { // The key refers to root directory of container. // Set the modification time for root to zero. return new FileMetadata(key, 0, defaultPermissionNoBlobMetadata(), BlobMaterialization.Implicit); } CloudBlobWrapper blob = getBlobReference(key); // Download attributes and return file metadata only if the blob // exists. if (null != blob && blob.exists(getInstrumentedContext())) { if (LOG.isDebugEnabled()) { LOG.debug("Found " + key + " as an explicit blob. Checking if it's a file or folder."); } // The blob exists, so capture the metadata from the blob // properties. blob.downloadAttributes(getInstrumentedContext()); BlobProperties properties = blob.getProperties(); if (retrieveFolderAttribute(blob)) { if (LOG.isDebugEnabled()) { LOG.debug(key + " is a folder blob."); } return new FileMetadata(key, properties.getLastModified().getTime(), getPermissionStatus(blob), BlobMaterialization.Explicit); } else { if (LOG.isDebugEnabled()) { LOG.debug(key + " is a normal blob."); } return new FileMetadata(key, // Always return denormalized key with metadata. getDataLength(blob, properties), properties.getLastModified().getTime(), getPermissionStatus(blob)); } } // There is no file with that key name, but maybe it is a folder. // Query the underlying folder/container to list the blobs stored // there under that key. // Iterable<ListBlobItem> objects = listRootBlobs(key, true, EnumSet.of(BlobListingDetails.METADATA), null, getInstrumentedContext()); // Check if the directory/container has the blob items. for (ListBlobItem blobItem : objects) { if (blobItem instanceof CloudBlockBlobWrapper || blobItem instanceof CloudPageBlobWrapper) { LOG.debug("Found blob as a directory-using this file under it to infer its properties " + blobItem.getUri()); blob = (CloudBlobWrapper) blobItem; // The key specifies a directory. Create a FileMetadata object which // specifies as such. BlobProperties properties = blob.getProperties(); return new FileMetadata(key, properties.getLastModified().getTime(), getPermissionStatus(blob), BlobMaterialization.Implicit); } } // Return to caller with a null metadata object. return null; } catch (Exception e) { // Re-throw the exception as an Azure storage exception. throw new AzureException(e); } } @Override public DataInputStream retrieve(String key) throws AzureException, IOException { try { // Check if a session exists, if not create a session with the // Azure storage server. if (null == storageInteractionLayer) { final String errMsg = String.format("Storage session expected for URI '%s' but does not exist.", sessionUri); throw new AssertionError(errMsg); } checkContainer(ContainerAccessType.PureRead); // Get blob reference and open the input buffer stream. CloudBlobWrapper blob = getBlobReference(key); BufferedInputStream inBufStream = new BufferedInputStream(openInputStream(blob)); // Return a data input stream. DataInputStream inDataStream = new DataInputStream(inBufStream); return inDataStream; } catch (Exception e) { // Re-throw as an Azure storage exception. throw new AzureException(e); } } @Override public DataInputStream retrieve(String key, long startByteOffset) throws AzureException, IOException { try { // Check if a session exists, if not create a session with the // Azure storage server. if (null == storageInteractionLayer) { final String errMsg = String.format("Storage session expected for URI '%s' but does not exist.", sessionUri); throw new AssertionError(errMsg); } checkContainer(ContainerAccessType.PureRead); // Get blob reference and open the input buffer stream. CloudBlobWrapper blob = getBlobReference(key); // Open input stream and seek to the start offset. InputStream in = blob.openInputStream(getDownloadOptions(), getInstrumentedContext(isConcurrentOOBAppendAllowed())); // Create a data input stream. DataInputStream inDataStream = new DataInputStream(in); // Skip bytes and ignore return value. This is okay // because if you try to skip too far you will be positioned // at the end and reads will not return data. inDataStream.skip(startByteOffset); return inDataStream; } catch (Exception e) { // Re-throw as an Azure storage exception. throw new AzureException(e); } } @Override public PartialListing list(String prefix, final int maxListingCount, final int maxListingDepth) throws IOException { return list(prefix, maxListingCount, maxListingDepth, null); } @Override public PartialListing list(String prefix, final int maxListingCount, final int maxListingDepth, String priorLastKey) throws IOException { return list(prefix, PATH_DELIMITER, maxListingCount, maxListingDepth, priorLastKey); } @Override public PartialListing listAll(String prefix, final int maxListingCount, final int maxListingDepth, String priorLastKey) throws IOException { return list(prefix, null, maxListingCount, maxListingDepth, priorLastKey); } /** * Searches the given list of {@link FileMetadata} objects for a directory * with the given key. * * @param list * The list to search. * @param key * The key to search for. * @return The wanted directory, or null if not found. */ private static FileMetadata getDirectoryInList(final Iterable<FileMetadata> list, String key) { for (FileMetadata current : list) { if (current.isDir() && current.getKey().equals(key)) { return current; } } return null; } private PartialListing list(String prefix, String delimiter, final int maxListingCount, final int maxListingDepth, String priorLastKey) throws IOException { try { checkContainer(ContainerAccessType.PureRead); if (0 < prefix.length() && !prefix.endsWith(PATH_DELIMITER)) { prefix += PATH_DELIMITER; } Iterable<ListBlobItem> objects; if (prefix.equals("/")) { objects = listRootBlobs(true); } else { objects = listRootBlobs(prefix, true); } ArrayList<FileMetadata> fileMetadata = new ArrayList<FileMetadata>(); for (ListBlobItem blobItem : objects) { // Check that the maximum listing count is not exhausted. // if (0 < maxListingCount && fileMetadata.size() >= maxListingCount) { break; } if (blobItem instanceof CloudBlockBlobWrapper || blobItem instanceof CloudPageBlobWrapper) { String blobKey = null; CloudBlobWrapper blob = (CloudBlobWrapper) blobItem; BlobProperties properties = blob.getProperties(); // Determine format of the blob name depending on whether an absolute // path is being used or not. blobKey = normalizeKey(blob); FileMetadata metadata; if (retrieveFolderAttribute(blob)) { metadata = new FileMetadata(blobKey, properties.getLastModified().getTime(), getPermissionStatus(blob), BlobMaterialization.Explicit); } else { metadata = new FileMetadata(blobKey, getDataLength(blob, properties), properties.getLastModified().getTime(), getPermissionStatus(blob)); } // Add the metadata to the list, but remove any existing duplicate // entries first that we may have added by finding nested files. FileMetadata existing = getDirectoryInList(fileMetadata, blobKey); if (existing != null) { fileMetadata.remove(existing); } fileMetadata.add(metadata); } else if (blobItem instanceof CloudBlobDirectoryWrapper) { CloudBlobDirectoryWrapper directory = (CloudBlobDirectoryWrapper) blobItem; // Determine format of directory name depending on whether an absolute // path is being used or not. // String dirKey = normalizeKey(directory); // Strip the last / if (dirKey.endsWith(PATH_DELIMITER)) { dirKey = dirKey.substring(0, dirKey.length() - 1); } // Reached the targeted listing depth. Return metadata for the // directory using default permissions. // // Note: Something smarter should be done about permissions. Maybe // inherit the permissions of the first non-directory blob. // Also, getting a proper value for last-modified is tricky. FileMetadata directoryMetadata = new FileMetadata(dirKey, 0, defaultPermissionNoBlobMetadata(), BlobMaterialization.Implicit); // Add the directory metadata to the list only if it's not already // there. if (getDirectoryInList(fileMetadata, dirKey) == null) { fileMetadata.add(directoryMetadata); } // Currently at a depth of one, decrement the listing depth for // sub-directories. buildUpList(directory, fileMetadata, maxListingCount, maxListingDepth - 1); } } // Note: Original code indicated that this may be a hack. priorLastKey = null; PartialListing listing = new PartialListing(priorLastKey, fileMetadata.toArray(new FileMetadata[] {}), 0 == fileMetadata.size() ? new String[] {} : new String[] { prefix }); return listing; } catch (Exception e) { // Re-throw as an Azure storage exception. // throw new AzureException(e); } } /** * Build up a metadata list of blobs in an Azure blob directory. This method * uses a in-order first traversal of blob directory structures to maintain * the sorted order of the blob names. * * @param aCloudBlobDirectory Azure blob directory * @param aFileMetadataList a list of file metadata objects for each * non-directory blob. * @param maxListingCount maximum length of the built up list. */ private void buildUpList(CloudBlobDirectoryWrapper aCloudBlobDirectory, ArrayList<FileMetadata> aFileMetadataList, final int maxListingCount, final int maxListingDepth) throws Exception { // Push the blob directory onto the stack. // AzureLinkedStack<Iterator<ListBlobItem>> dirIteratorStack = new AzureLinkedStack<Iterator<ListBlobItem>>(); Iterable<ListBlobItem> blobItems = aCloudBlobDirectory.listBlobs(null, false, EnumSet.of(BlobListingDetails.METADATA), null, getInstrumentedContext()); Iterator<ListBlobItem> blobItemIterator = blobItems.iterator(); if (0 == maxListingDepth || 0 == maxListingCount) { // Recurrence depth and listing count are already exhausted. Return // immediately. return; } // The directory listing depth is unbounded if the maximum listing depth // is negative. final boolean isUnboundedDepth = (maxListingDepth < 0); // Reset the current directory listing depth. int listingDepth = 1; // Loop until all directories have been traversed in-order. Loop only // the following conditions are satisfied: // (1) The stack is not empty, and // (2) maxListingCount > 0 implies that the number of items in the // metadata list is less than the max listing count. while (null != blobItemIterator && (maxListingCount <= 0 || aFileMetadataList.size() < maxListingCount)) { while (blobItemIterator.hasNext()) { // Check if the count of items on the list exhausts the maximum // listing count. // if (0 < maxListingCount && aFileMetadataList.size() >= maxListingCount) { break; } ListBlobItem blobItem = blobItemIterator.next(); // Add the file metadata to the list if this is not a blob // directory item. // if (blobItem instanceof CloudBlockBlobWrapper || blobItem instanceof CloudPageBlobWrapper) { String blobKey = null; CloudBlobWrapper blob = (CloudBlobWrapper) blobItem; BlobProperties properties = blob.getProperties(); // Determine format of the blob name depending on whether an absolute // path is being used or not. blobKey = normalizeKey(blob); FileMetadata metadata; if (retrieveFolderAttribute(blob)) { metadata = new FileMetadata(blobKey, properties.getLastModified().getTime(), getPermissionStatus(blob), BlobMaterialization.Explicit); } else { metadata = new FileMetadata(blobKey, getDataLength(blob, properties), properties.getLastModified().getTime(), getPermissionStatus(blob)); } // Add the directory metadata to the list only if it's not already // there. FileMetadata existing = getDirectoryInList(aFileMetadataList, blobKey); if (existing != null) { aFileMetadataList.remove(existing); } aFileMetadataList.add(metadata); } else if (blobItem instanceof CloudBlobDirectoryWrapper) { CloudBlobDirectoryWrapper directory = (CloudBlobDirectoryWrapper) blobItem; // This is a directory blob, push the current iterator onto // the stack of iterators and start iterating through the current // directory. if (isUnboundedDepth || maxListingDepth > listingDepth) { // Push the current directory on the stack and increment the listing // depth. dirIteratorStack.push(blobItemIterator); ++listingDepth; // The current blob item represents the new directory. Get // an iterator for this directory and continue by iterating through // this directory. blobItems = directory.listBlobs(null, false, EnumSet.noneOf(BlobListingDetails.class), null, getInstrumentedContext()); blobItemIterator = blobItems.iterator(); } else { // Determine format of directory name depending on whether an // absolute path is being used or not. String dirKey = normalizeKey(directory); if (getDirectoryInList(aFileMetadataList, dirKey) == null) { // Reached the targeted listing depth. Return metadata for the // directory using default permissions. // // Note: Something smarter should be done about permissions. Maybe // inherit the permissions of the first non-directory blob. // Also, getting a proper value for last-modified is tricky. // FileMetadata directoryMetadata = new FileMetadata(dirKey, 0, defaultPermissionNoBlobMetadata(), BlobMaterialization.Implicit); // Add the directory metadata to the list. aFileMetadataList.add(directoryMetadata); } } } } // Traversal of directory tree // Check if the iterator stack is empty. If it is set the next blob // iterator to null. This will act as a terminator for the for-loop. // Otherwise pop the next iterator from the stack and continue looping. // if (dirIteratorStack.isEmpty()) { blobItemIterator = null; } else { // Pop the next directory item from the stack and decrement the // depth. blobItemIterator = dirIteratorStack.pop(); --listingDepth; // Assertion: Listing depth should not be less than zero. if (listingDepth < 0) { throw new AssertionError("Non-negative listing depth expected"); } } } } /** * Return the actual data length of the blob with the specified properties. * If it is a page blob, you can't rely on the length from the properties * argument and you must get it from the file. Otherwise, you can. */ private long getDataLength(CloudBlobWrapper blob, BlobProperties properties) throws AzureException { if (blob instanceof CloudPageBlobWrapper) { try { return PageBlobInputStream.getPageBlobDataSize((CloudPageBlobWrapper) blob, getInstrumentedContext(isConcurrentOOBAppendAllowed())); } catch (Exception e) { throw new AzureException("Unexpected exception getting page blob actual data size.", e); } } return properties.getLength(); } /** * Deletes the given blob, taking special care that if we get a * blob-not-found exception upon retrying the operation, we just * swallow the error since what most probably happened is that * the first operation succeeded on the server. * @param blob The blob to delete. * @param lease Azure blob lease, or null if no lease is to be used. * @throws StorageException */ private void safeDelete(CloudBlobWrapper blob, SelfRenewingLease lease) throws StorageException { OperationContext operationContext = getInstrumentedContext(); try { blob.delete(operationContext, lease); } catch (StorageException e) { // On exception, check that if: // 1. It's a BlobNotFound exception AND // 2. It got there after one-or-more retries THEN // we swallow the exception. if (e.getErrorCode() != null && e.getErrorCode().equals("BlobNotFound") && operationContext.getRequestResults().size() > 1 && operationContext.getRequestResults().get(0).getException() != null) { if (LOG.isDebugEnabled()) { LOG.debug("Swallowing delete exception on retry: " + e.getMessage()); } return; } else { throw e; } } finally { if (lease != null) { lease.free(); } } } @Override public void delete(String key, SelfRenewingLease lease) throws IOException { try { if (checkContainer(ContainerAccessType.ReadThenWrite) == ContainerState.DoesntExist) { // Container doesn't exist, no need to do anything return; } // Get the blob reference and delete it. CloudBlobWrapper blob = getBlobReference(key); if (blob.exists(getInstrumentedContext())) { safeDelete(blob, lease); } } catch (Exception e) { // Re-throw as an Azure storage exception. throw new AzureException(e); } } @Override public void delete(String key) throws IOException { delete(key, null); } @Override public void rename(String srcKey, String dstKey) throws IOException { rename(srcKey, dstKey, false, null); } @Override public void rename(String srcKey, String dstKey, boolean acquireLease, SelfRenewingLease existingLease) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("Moving " + srcKey + " to " + dstKey); } if (acquireLease && existingLease != null) { throw new IOException("Cannot acquire new lease if one already exists."); } try { // Attempts rename may occur before opening any streams so first, // check if a session exists, if not create a session with the Azure // storage server. if (null == storageInteractionLayer) { final String errMsg = String.format("Storage session expected for URI '%s' but does not exist.", sessionUri); throw new AssertionError(errMsg); } checkContainer(ContainerAccessType.ReadThenWrite); // Get the source blob and assert its existence. If the source key // needs to be normalized then normalize it. // CloudBlobWrapper srcBlob = getBlobReference(srcKey); if (!srcBlob.exists(getInstrumentedContext())) { throw new AzureException("Source blob " + srcKey + " does not exist."); } /** * Conditionally get a lease on the source blob to prevent other writers * from changing it. This is used for correctness in HBase when log files * are renamed. It generally should do no harm other than take a little * more time for other rename scenarios. When the HBase master renames a * log file folder, the lease locks out other writers. This * prevents a region server that the master thinks is dead, but is still * alive, from committing additional updates. This is different than * when HBase runs on HDFS, where the region server recovers the lease * on a log file, to gain exclusive access to it, before it splits it. */ SelfRenewingLease lease = null; if (acquireLease) { lease = srcBlob.acquireLease(); } else if (existingLease != null) { lease = existingLease; } // Get the destination blob. The destination key always needs to be // normalized. // CloudBlobWrapper dstBlob = getBlobReference(dstKey); // Rename the source blob to the destination blob by copying it to // the destination blob then deleting it. // // Copy blob operation in Azure storage is very costly. It will be highly // likely throttled during Azure storage gc. Short term fix will be using // a more intensive exponential retry policy when the cluster is getting // throttled. try { dstBlob.startCopyFromBlob(srcBlob, null, getInstrumentedContext()); } catch (StorageException se) { if (se.getErrorCode().equals(StorageErrorCode.SERVER_BUSY.toString())) { int copyBlobMinBackoff = sessionConfiguration.getInt(KEY_COPYBLOB_MIN_BACKOFF_INTERVAL, DEFAULT_COPYBLOB_MIN_BACKOFF_INTERVAL); int copyBlobMaxBackoff = sessionConfiguration.getInt(KEY_COPYBLOB_MAX_BACKOFF_INTERVAL, DEFAULT_COPYBLOB_MAX_BACKOFF_INTERVAL); int copyBlobDeltaBackoff = sessionConfiguration.getInt(KEY_COPYBLOB_BACKOFF_INTERVAL, DEFAULT_COPYBLOB_BACKOFF_INTERVAL); int copyBlobMaxRetries = sessionConfiguration.getInt(KEY_COPYBLOB_MAX_IO_RETRIES, DEFAULT_COPYBLOB_MAX_RETRY_ATTEMPTS); BlobRequestOptions options = new BlobRequestOptions(); options.setRetryPolicyFactory(new RetryExponentialRetry(copyBlobMinBackoff, copyBlobDeltaBackoff, copyBlobMaxBackoff, copyBlobMaxRetries)); dstBlob.startCopyFromBlob(srcBlob, options, getInstrumentedContext()); } else { throw se; } } waitForCopyToComplete(dstBlob, getInstrumentedContext()); safeDelete(srcBlob, lease); } catch (StorageException e) { // Re-throw exception as an Azure storage exception. throw new AzureException(e); } catch (URISyntaxException e) { // Re-throw exception as an Azure storage exception. throw new AzureException(e); } } private void waitForCopyToComplete(CloudBlobWrapper blob, OperationContext opContext) { boolean copyInProgress = true; while (copyInProgress) { try { blob.downloadAttributes(opContext); } catch (StorageException se) { } // test for null because mocked filesystem doesn't know about copystates yet. copyInProgress = (blob.getCopyState() != null && blob.getCopyState().getStatus() == CopyStatus.PENDING); if (copyInProgress) { try { Thread.sleep(1000); } catch (InterruptedException ie) { //ignore } } } } /** * Changes the permission status on the given key. */ @Override public void changePermissionStatus(String key, PermissionStatus newPermission) throws AzureException { try { checkContainer(ContainerAccessType.ReadThenWrite); CloudBlobWrapper blob = getBlobReference(key); blob.downloadAttributes(getInstrumentedContext()); storePermissionStatus(blob, newPermission); blob.uploadMetadata(getInstrumentedContext()); } catch (Exception e) { throw new AzureException(e); } } @Override public void purge(String prefix) throws IOException { try { // Attempts to purge may occur before opening any streams so first, // check if a session exists, if not create a session with the Azure // storage server. if (null == storageInteractionLayer) { final String errMsg = String.format("Storage session expected for URI '%s' but does not exist.", sessionUri); throw new AssertionError(errMsg); } if (checkContainer(ContainerAccessType.ReadThenWrite) == ContainerState.DoesntExist) { // Container doesn't exist, no need to do anything. return; } // Get all blob items with the given prefix from the container and delete // them. Iterable<ListBlobItem> objects = listRootBlobs(prefix, false); for (ListBlobItem blobItem : objects) { ((CloudBlob) blobItem).delete(DeleteSnapshotsOption.NONE, null, null, getInstrumentedContext()); } } catch (Exception e) { // Re-throw as an Azure storage exception. // throw new AzureException(e); } } /** * Get a lease on the blob identified by key. This lease will be renewed * indefinitely by a background thread. */ @Override public SelfRenewingLease acquireLease(String key) throws AzureException { LOG.debug("acquiring lease on " + key); try { checkContainer(ContainerAccessType.ReadThenWrite); CloudBlobWrapper blob = getBlobReference(key); return blob.acquireLease(); } catch (Exception e) { // Caught exception while attempting to get lease. Re-throw as an // Azure storage exception. throw new AzureException(e); } } @Override public void updateFolderLastModifiedTime(String key, Date lastModified, SelfRenewingLease folderLease) throws AzureException { try { checkContainer(ContainerAccessType.ReadThenWrite); CloudBlobWrapper blob = getBlobReference(key); //setLastModified function is not available in 2.0.0 version. blob.uploadProperties automatically updates last modified //timestamp to current time blob.uploadProperties(getInstrumentedContext(), folderLease); } catch (Exception e) { // Caught exception while attempting to update the properties. Re-throw as an // Azure storage exception. throw new AzureException(e); } } @Override public void updateFolderLastModifiedTime(String key, SelfRenewingLease folderLease) throws AzureException { final Calendar lastModifiedCalendar = Calendar.getInstance(Utility.LOCALE_US); lastModifiedCalendar.setTimeZone(Utility.UTC_ZONE); Date lastModified = lastModifiedCalendar.getTime(); updateFolderLastModifiedTime(key, lastModified, folderLease); } @Override public void dump() throws IOException { } @Override public void close() { if (bandwidthGaugeUpdater != null) { bandwidthGaugeUpdater.close(); bandwidthGaugeUpdater = null; } } // Finalizer to ensure complete shutdown @Override protected void finalize() throws Throwable { LOG.debug("finalize() called"); close(); super.finalize(); } }