Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.fs.azure; import java.io.DataInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.Charset; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.EnumSet; import java.util.Iterator; import java.util.Set; import java.util.TimeZone; import java.util.TreeSet; import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.BufferedFSInputStream; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FSInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation; import org.apache.hadoop.fs.azure.metrics.AzureFileSystemMetricsSystem; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.fs.azure.AzureException; import org.apache.hadoop.fs.azure.StorageInterface.CloudBlobWrapper; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Progressable; import org.codehaus.jackson.JsonNode; import org.codehaus.jackson.JsonParseException; import org.codehaus.jackson.JsonParser; import org.codehaus.jackson.map.JsonMappingException; import org.codehaus.jackson.map.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import com.microsoft.azure.storage.AccessCondition; import com.microsoft.azure.storage.OperationContext; import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.blob.CloudBlob; import com.microsoft.azure.storage.core.*; /** * A {@link FileSystem} for reading and writing files stored on <a * href="http://store.azure.com/">Windows Azure</a>. This implementation is * blob-based and stores files on Azure in their native form so they can be read * by other Azure tools. */ @InterfaceAudience.Public @InterfaceStability.Stable public class NativeAzureFileSystem extends FileSystem { private static final int USER_WX_PERMISION = 0300; /** * A description of a folder rename operation, including the source and * destination keys, and descriptions of the files in the source folder. */ public static class FolderRenamePending { private SelfRenewingLease folderLease; private String srcKey; private String dstKey; private FileMetadata[] fileMetadata = null; // descriptions of source files private ArrayList<String> fileStrings = null; private NativeAzureFileSystem fs; private static final int MAX_RENAME_PENDING_FILE_SIZE = 10000000; private static final int FORMATTING_BUFFER = 10000; private boolean committed; public static final String SUFFIX = "-RenamePending.json"; // Prepare in-memory information needed to do or redo a folder rename. public FolderRenamePending(String srcKey, String dstKey, SelfRenewingLease lease, NativeAzureFileSystem fs) throws IOException { this.srcKey = srcKey; this.dstKey = dstKey; this.folderLease = lease; this.fs = fs; ArrayList<FileMetadata> fileMetadataList = new ArrayList<FileMetadata>(); // List all the files in the folder. String priorLastKey = null; do { PartialListing listing = fs.getStoreInterface().listAll(srcKey, AZURE_LIST_ALL, AZURE_UNBOUNDED_DEPTH, priorLastKey); for (FileMetadata file : listing.getFiles()) { fileMetadataList.add(file); } priorLastKey = listing.getPriorLastKey(); } while (priorLastKey != null); fileMetadata = fileMetadataList.toArray(new FileMetadata[fileMetadataList.size()]); this.committed = true; } // Prepare in-memory information needed to do or redo folder rename from // a -RenamePending.json file read from storage. This constructor is to use during // redo processing. public FolderRenamePending(Path redoFile, NativeAzureFileSystem fs) throws IllegalArgumentException, IOException { this.fs = fs; // open redo file Path f = redoFile; FSDataInputStream input = fs.open(f); byte[] bytes = new byte[MAX_RENAME_PENDING_FILE_SIZE]; int l = input.read(bytes); if (l < 0) { throw new IOException("Error reading pending rename file contents -- no data available"); } if (l == MAX_RENAME_PENDING_FILE_SIZE) { throw new IOException( "Error reading pending rename file contents -- " + "maximum file size exceeded"); } String contents = new String(bytes, 0, l, Charset.forName("UTF-8")); // parse the JSON ObjectMapper objMapper = new ObjectMapper(); objMapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true); JsonNode json = null; try { json = objMapper.readValue(contents, JsonNode.class); this.committed = true; } catch (JsonMappingException e) { // The -RedoPending.json file is corrupted, so we assume it was // not completely written // and the redo operation did not commit. this.committed = false; } catch (JsonParseException e) { this.committed = false; } catch (IOException e) { this.committed = false; } if (!this.committed) { LOG.error("Deleting corruped rename pending file " + redoFile + "\n" + contents); // delete the -RenamePending.json file fs.delete(redoFile, false); return; } // initialize this object's fields ArrayList<String> fileStrList = new ArrayList<String>(); JsonNode oldFolderName = json.get("OldFolderName"); JsonNode newFolderName = json.get("NewFolderName"); if (oldFolderName == null || newFolderName == null) { this.committed = false; } else { this.srcKey = oldFolderName.getTextValue(); this.dstKey = newFolderName.getTextValue(); if (this.srcKey == null || this.dstKey == null) { this.committed = false; } else { JsonNode fileList = json.get("FileList"); if (fileList == null) { this.committed = false; } else { for (int i = 0; i < fileList.size(); i++) { fileStrList.add(fileList.get(i).getTextValue()); } } } } this.fileStrings = fileStrList; } public FileMetadata[] getFiles() { return fileMetadata; } public SelfRenewingLease getFolderLease() { return folderLease; } /** * Write to disk the information needed to redo folder rename, * in JSON format. The file name will be * {@code wasb://<sourceFolderPrefix>/folderName-RenamePending.json} * The file format will be: * <pre>{@code * { * FormatVersion: "1.0", * OperationTime: "<YYYY-MM-DD HH:MM:SS.MMM>", * OldFolderName: "<key>", * NewFolderName: "<key>", * FileList: [ <string> , <string> , ... ] * } * * Here's a sample: * { * FormatVersion: "1.0", * OperationUTCTime: "2014-07-01 23:50:35.572", * OldFolderName: "user/ehans/folderToRename", * NewFolderName: "user/ehans/renamedFolder", * FileList: [ * "innerFile", * "innerFile2" * ] * } }</pre> * @throws IOException */ public void writeFile(FileSystem fs) throws IOException { Path path = getRenamePendingFilePath(); if (LOG.isDebugEnabled()) { LOG.debug("Preparing to write atomic rename state to " + path.toString()); } OutputStream output = null; String contents = makeRenamePendingFileContents(); // Write file. try { output = fs.create(path); output.write(contents.getBytes(Charset.forName("UTF-8"))); } catch (IOException e) { throw new IOException( "Unable to write RenamePending file for folder rename from " + srcKey + " to " + dstKey, e); } finally { IOUtils.cleanup(LOG, output); } } /** * Return the contents of the JSON file to represent the operations * to be performed for a folder rename. */ public String makeRenamePendingFileContents() { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); sdf.setTimeZone(TimeZone.getTimeZone("UTC")); String time = sdf.format(new Date()); // Make file list string StringBuilder builder = new StringBuilder(); builder.append("[\n"); for (int i = 0; i != fileMetadata.length; i++) { if (i > 0) { builder.append(",\n"); } builder.append(" "); String noPrefix = StringUtils.removeStart(fileMetadata[i].getKey(), srcKey + "/"); // Quote string file names, escaping any possible " characters or other // necessary characters in the name. builder.append(quote(noPrefix)); if (builder.length() >= MAX_RENAME_PENDING_FILE_SIZE - FORMATTING_BUFFER) { // Give up now to avoid using too much memory. LOG.error("Internal error: Exceeded maximum rename pending file size of " + MAX_RENAME_PENDING_FILE_SIZE + " bytes."); // return some bad JSON with an error message to make it human readable return "exceeded maximum rename pending file size"; } } builder.append("\n ]"); String fileList = builder.toString(); // Make file contents as a string. Again, quote file names, escaping // characters as appropriate. String contents = "{\n" + " FormatVersion: \"1.0\",\n" + " OperationUTCTime: \"" + time + "\",\n" + " OldFolderName: " + quote(srcKey) + ",\n" + " NewFolderName: " + quote(dstKey) + ",\n" + " FileList: " + fileList + "\n" + "}\n"; return contents; } /** * This is an exact copy of org.codehaus.jettison.json.JSONObject.quote * method. * * Produce a string in double quotes with backslash sequences in all the * right places. A backslash will be inserted within </, allowing JSON * text to be delivered in HTML. In JSON text, a string cannot contain a * control character or an unescaped quote or backslash. * @param string A String * @return A String correctly formatted for insertion in a JSON text. */ private String quote(String string) { if (string == null || string.length() == 0) { return "\"\""; } char c = 0; int i; int len = string.length(); StringBuilder sb = new StringBuilder(len + 4); String t; sb.append('"'); for (i = 0; i < len; i += 1) { c = string.charAt(i); switch (c) { case '\\': case '"': sb.append('\\'); sb.append(c); break; case '/': sb.append('\\'); sb.append(c); break; case '\b': sb.append("\\b"); break; case '\t': sb.append("\\t"); break; case '\n': sb.append("\\n"); break; case '\f': sb.append("\\f"); break; case '\r': sb.append("\\r"); break; default: if (c < ' ') { t = "000" + Integer.toHexString(c); sb.append("\\u" + t.substring(t.length() - 4)); } else { sb.append(c); } } } sb.append('"'); return sb.toString(); } public String getSrcKey() { return srcKey; } public String getDstKey() { return dstKey; } public FileMetadata getSourceMetadata() throws IOException { return fs.getStoreInterface().retrieveMetadata(srcKey); } /** * Execute a folder rename. This is the execution path followed * when everything is working normally. See redo() for the alternate * execution path for the case where we're recovering from a folder rename * failure. * @throws IOException */ public void execute() throws IOException { for (FileMetadata file : this.getFiles()) { // Rename all materialized entries under the folder to point to the // final destination. if (file.getBlobMaterialization() == BlobMaterialization.Explicit) { String srcName = file.getKey(); String suffix = srcName.substring((this.getSrcKey()).length()); String dstName = this.getDstKey() + suffix; // Rename gets exclusive access (via a lease) for files // designated for atomic rename. // The main use case is for HBase write-ahead log (WAL) and data // folder processing correctness. See the rename code for details. boolean acquireLease = fs.getStoreInterface().isAtomicRenameKey(srcName); fs.getStoreInterface().rename(srcName, dstName, acquireLease, null); } } // Rename the source folder 0-byte root file itself. FileMetadata srcMetadata2 = this.getSourceMetadata(); if (srcMetadata2.getBlobMaterialization() == BlobMaterialization.Explicit) { // It already has a lease on it from the "prepare" phase so there's no // need to get one now. Pass in existing lease to allow file delete. fs.getStoreInterface().rename(this.getSrcKey(), this.getDstKey(), false, folderLease); } // Update the last-modified time of the parent folders of both source and // destination. fs.updateParentFolderLastModifiedTime(srcKey); fs.updateParentFolderLastModifiedTime(dstKey); } /** Clean up after execution of rename. * @throws IOException */ public void cleanup() throws IOException { if (fs.getStoreInterface().isAtomicRenameKey(srcKey)) { // Remove RenamePending file fs.delete(getRenamePendingFilePath(), false); // Freeing source folder lease is not necessary since the source // folder file was deleted. } } private Path getRenamePendingFilePath() { String fileName = srcKey + SUFFIX; Path fileNamePath = keyToPath(fileName); Path path = fs.makeAbsolute(fileNamePath); return path; } /** * Recover from a folder rename failure by redoing the intended work, * as recorded in the -RenamePending.json file. * * @throws IOException */ public void redo() throws IOException { if (!committed) { // Nothing to do. The -RedoPending.json file should have already been // deleted. return; } // Try to get a lease on source folder to block concurrent access to it. // It may fail if the folder is already gone. We don't check if the // source exists explicitly because that could recursively trigger redo // and give an infinite recursion. SelfRenewingLease lease = null; boolean sourceFolderGone = false; try { lease = fs.leaseSourceFolder(srcKey); } catch (AzureException e) { // If the source folder was not found then somebody probably // raced with us and finished the rename first, or the // first rename failed right before deleting the rename pending // file. String errorCode = ""; try { StorageException se = (StorageException) e.getCause(); errorCode = se.getErrorCode(); } catch (Exception e2) { ; // do nothing -- could not get errorCode } if (errorCode.equals("BlobNotFound")) { sourceFolderGone = true; } else { throw new IOException("Unexpected error when trying to lease source folder name during " + "folder rename redo", e); } } if (!sourceFolderGone) { // Make sure the target folder exists. Path dst = fullPath(dstKey); if (!fs.exists(dst)) { fs.mkdirs(dst); } // For each file inside the folder to be renamed, // make sure it has been renamed. for (String fileName : fileStrings) { finishSingleFileRename(fileName); } // Remove the source folder. Don't check explicitly if it exists, // to avoid triggering redo recursively. try { fs.getStoreInterface().delete(srcKey, lease); } catch (Exception e) { LOG.info("Unable to delete source folder during folder rename redo. " + "If the source folder is already gone, this is not an error " + "condition. Continuing with redo.", e); } // Update the last-modified time of the parent folders of both source // and destination. fs.updateParentFolderLastModifiedTime(srcKey); fs.updateParentFolderLastModifiedTime(dstKey); } // Remove the -RenamePending.json file. fs.delete(getRenamePendingFilePath(), false); } // See if the source file is still there, and if it is, rename it. private void finishSingleFileRename(String fileName) throws IOException { Path srcFile = fullPath(srcKey, fileName); Path dstFile = fullPath(dstKey, fileName); boolean srcExists = fs.exists(srcFile); boolean dstExists = fs.exists(dstFile); if (srcExists && !dstExists) { // Rename gets exclusive access (via a lease) for HBase write-ahead log // (WAL) file processing correctness. See the rename code for details. String srcName = fs.pathToKey(srcFile); String dstName = fs.pathToKey(dstFile); fs.getStoreInterface().rename(srcName, dstName, true, null); } else if (srcExists && dstExists) { // Get a lease on source to block write access. String srcName = fs.pathToKey(srcFile); SelfRenewingLease lease = fs.acquireLease(srcFile); // Delete the file. This will free the lease too. fs.getStoreInterface().delete(srcName, lease); } else if (!srcExists && dstExists) { // The rename already finished, so do nothing. ; } else { throw new IOException("Attempting to complete rename of file " + srcKey + "/" + fileName + " during folder rename redo, and file was not found in source " + "or destination."); } } // Return an absolute path for the specific fileName within the folder // specified by folderKey. private Path fullPath(String folderKey, String fileName) { return new Path(new Path(fs.getUri()), "/" + folderKey + "/" + fileName); } private Path fullPath(String fileKey) { return new Path(new Path(fs.getUri()), "/" + fileKey); } } private static final String TRAILING_PERIOD_PLACEHOLDER = "[[.]]"; private static final Pattern TRAILING_PERIOD_PLACEHOLDER_PATTERN = Pattern.compile("\\[\\[\\.\\]\\](?=$|/)"); private static final Pattern TRAILING_PERIOD_PATTERN = Pattern.compile("\\.(?=$|/)"); @Override public String getScheme() { return "wasb"; } /** * <p> * A {@link FileSystem} for reading and writing files stored on <a * href="http://store.azure.com/">Windows Azure</a>. This implementation is * blob-based and stores files on Azure in their native form so they can be read * by other Azure tools. This implementation uses HTTPS for secure network communication. * </p> */ public static class Secure extends NativeAzureFileSystem { @Override public String getScheme() { return "wasbs"; } } public static final Log LOG = LogFactory.getLog(NativeAzureFileSystem.class); static final String AZURE_BLOCK_SIZE_PROPERTY_NAME = "fs.azure.block.size"; /** * The time span in seconds before which we consider a temp blob to be * dangling (not being actively uploaded to) and up for reclamation. * * So e.g. if this is 60, then any temporary blobs more than a minute old * would be considered dangling. */ static final String AZURE_TEMP_EXPIRY_PROPERTY_NAME = "fs.azure.fsck.temp.expiry.seconds"; private static final int AZURE_TEMP_EXPIRY_DEFAULT = 3600; static final String PATH_DELIMITER = Path.SEPARATOR; static final String AZURE_TEMP_FOLDER = "_$azuretmpfolder$"; private static final int AZURE_LIST_ALL = -1; private static final int AZURE_UNBOUNDED_DEPTH = -1; private static final long MAX_AZURE_BLOCK_SIZE = 512 * 1024 * 1024L; /** * The configuration property that determines which group owns files created * in WASB. */ private static final String AZURE_DEFAULT_GROUP_PROPERTY_NAME = "fs.azure.permissions.supergroup"; /** * The default value for fs.azure.permissions.supergroup. Chosen as the same * default as DFS. */ static final String AZURE_DEFAULT_GROUP_DEFAULT = "supergroup"; static final String AZURE_BLOCK_LOCATION_HOST_PROPERTY_NAME = "fs.azure.block.location.impersonatedhost"; private static final String AZURE_BLOCK_LOCATION_HOST_DEFAULT = "localhost"; static final String AZURE_RINGBUFFER_CAPACITY_PROPERTY_NAME = "fs.azure.ring.buffer.capacity"; static final String AZURE_OUTPUT_STREAM_BUFFER_SIZE_PROPERTY_NAME = "fs.azure.output.stream.buffer.size"; public static final String SKIP_AZURE_METRICS_PROPERTY_NAME = "fs.azure.skip.metrics"; private class NativeAzureFsInputStream extends FSInputStream { private InputStream in; private final String key; private long pos = 0; private boolean closed = false; private boolean isPageBlob; // File length, valid only for streams over block blobs. private long fileLength; public NativeAzureFsInputStream(DataInputStream in, String key, long fileLength) { this.in = in; this.key = key; this.isPageBlob = store.isPageBlobKey(key); this.fileLength = fileLength; } /** * Return the size of the remaining available bytes * if the size is less than or equal to {@link Integer#MAX_VALUE}, * otherwise, return {@link Integer#MAX_VALUE}. * * This is to match the behavior of DFSInputStream.available(), * which some clients may rely on (HBase write-ahead log reading in * particular). */ @Override public synchronized int available() throws IOException { if (isPageBlob) { return in.available(); } else { if (closed) { throw new IOException("Stream closed"); } final long remaining = this.fileLength - pos; return remaining <= Integer.MAX_VALUE ? (int) remaining : Integer.MAX_VALUE; } } /* * Reads the next byte of data from the input stream. The value byte is * returned as an integer in the range 0 to 255. If no byte is available * because the end of the stream has been reached, the value -1 is returned. * This method blocks until input data is available, the end of the stream * is detected, or an exception is thrown. * * @returns int An integer corresponding to the byte read. */ @Override public synchronized int read() throws IOException { int result = 0; result = in.read(); if (result != -1) { pos++; if (statistics != null) { statistics.incrementBytesRead(1); } } // Return to the caller with the result. // return result; } /* * Reads up to len bytes of data from the input stream into an array of * bytes. An attempt is made to read as many as len bytes, but a smaller * number may be read. The number of bytes actually read is returned as an * integer. This method blocks until input data is available, end of file is * detected, or an exception is thrown. If len is zero, then no bytes are * read and 0 is returned; otherwise, there is an attempt to read at least * one byte. If no byte is available because the stream is at end of file, * the value -1 is returned; otherwise, at least one byte is read and stored * into b. * * @param b -- the buffer into which data is read * * @param off -- the start offset in the array b at which data is written * * @param len -- the maximum number of bytes read * * @ returns int The total number of byes read into the buffer, or -1 if * there is no more data because the end of stream is reached. */ @Override public synchronized int read(byte[] b, int off, int len) throws IOException { int result = 0; result = in.read(b, off, len); if (result > 0) { pos += result; } if (null != statistics) { statistics.incrementBytesRead(result); } // Return to the caller with the result. return result; } @Override public void close() throws IOException { in.close(); closed = true; } @Override public synchronized void seek(long pos) throws IOException { in.close(); in = store.retrieve(key); this.pos = in.skip(pos); if (LOG.isDebugEnabled()) { LOG.debug(String.format("Seek to position %d. Bytes skipped %d", pos, this.pos)); } } @Override public synchronized long getPos() throws IOException { return pos; } @Override public boolean seekToNewSource(long targetPos) throws IOException { return false; } } private class NativeAzureFsOutputStream extends OutputStream { // We should not override flush() to actually close current block and flush // to DFS, this will break applications that assume flush() is a no-op. // Applications are advised to use Syncable.hflush() for that purpose. // NativeAzureFsOutputStream needs to implement Syncable if needed. private String key; private String keyEncoded; private OutputStream out; public NativeAzureFsOutputStream(OutputStream out, String aKey, String anEncodedKey) throws IOException { // Check input arguments. The output stream should be non-null and the // keys // should be valid strings. if (null == out) { throw new IllegalArgumentException("Illegal argument: the output stream is null."); } if (null == aKey || 0 == aKey.length()) { throw new IllegalArgumentException("Illegal argument the key string is null or empty"); } if (null == anEncodedKey || 0 == anEncodedKey.length()) { throw new IllegalArgumentException("Illegal argument the encoded key string is null or empty"); } // Initialize the member variables with the incoming parameters. this.out = out; setKey(aKey); setEncodedKey(anEncodedKey); } @Override public synchronized void close() throws IOException { if (out != null) { // Close the output stream and decode the key for the output stream // before returning to the caller. // out.close(); restoreKey(); out = null; } } /** * Writes the specified byte to this output stream. The general contract for * write is that one byte is written to the output stream. The byte to be * written is the eight low-order bits of the argument b. The 24 high-order * bits of b are ignored. * * @param b * 32-bit integer of block of 4 bytes */ @Override public void write(int b) throws IOException { out.write(b); } /** * Writes b.length bytes from the specified byte array to this output * stream. The general contract for write(b) is that it should have exactly * the same effect as the call write(b, 0, b.length). * * @param b * Block of bytes to be written to the output stream. */ @Override public void write(byte[] b) throws IOException { out.write(b); } /** * Writes <code>len</code> from the specified byte array starting at offset * <code>off</code> to the output stream. The general contract for write(b, * off, len) is that some of the bytes in the array <code> * b</code b> are written to the output stream in order; element * <code>b[off]</code> is the first byte written and * <code>b[off+len-1]</code> is the last byte written by this operation. * * @param b * Byte array to be written. * @param off * Write this offset in stream. * @param len * Number of bytes to be written. */ @Override public void write(byte[] b, int off, int len) throws IOException { out.write(b, off, len); } /** * Get the blob name. * * @return String Blob name. */ public String getKey() { return key; } /** * Set the blob name. * * @param key * Blob name. */ public void setKey(String key) { this.key = key; } /** * Get the blob name. * * @return String Blob name. */ public String getEncodedKey() { return keyEncoded; } /** * Set the blob name. * * @param anEncodedKey * Blob name. */ public void setEncodedKey(String anEncodedKey) { this.keyEncoded = anEncodedKey; } /** * Restore the original key name from the m_key member variable. Note: The * output file stream is created with an encoded blob store key to guarantee * load balancing on the front end of the Azure storage partition servers. * The create also includes the name of the original key value which is * stored in the m_key member variable. This method should only be called * when the stream is closed. */ private void restoreKey() throws IOException { store.rename(getEncodedKey(), getKey()); } } private URI uri; private NativeFileSystemStore store; private AzureNativeFileSystemStore actualStore; private Path workingDir; private long blockSize = MAX_AZURE_BLOCK_SIZE; private AzureFileSystemInstrumentation instrumentation; private String metricsSourceName; private boolean isClosed = false; private static boolean suppressRetryPolicy = false; // A counter to create unique (within-process) names for my metrics sources. private static AtomicInteger metricsSourceNameCounter = new AtomicInteger(); public NativeAzureFileSystem() { // set store in initialize() } public NativeAzureFileSystem(NativeFileSystemStore store) { this.store = store; } /** * Suppress the default retry policy for the Storage, useful in unit tests to * test negative cases without waiting forever. */ @VisibleForTesting static void suppressRetryPolicy() { suppressRetryPolicy = true; } /** * Undo the effect of suppressRetryPolicy. */ @VisibleForTesting static void resumeRetryPolicy() { suppressRetryPolicy = false; } /** * Creates a new metrics source name that's unique within this process. */ @VisibleForTesting public static String newMetricsSourceName() { int number = metricsSourceNameCounter.incrementAndGet(); final String baseName = "AzureFileSystemMetrics"; if (number == 1) { // No need for a suffix for the first one return baseName; } else { return baseName + number; } } /** * Checks if the given URI scheme is a scheme that's affiliated with the Azure * File System. * * @param scheme * The URI scheme. * @return true iff it's an Azure File System URI scheme. */ private static boolean isWasbScheme(String scheme) { // The valid schemes are: asv (old name), asvs (old name over HTTPS), // wasb (new name), wasbs (new name over HTTPS). return scheme != null && (scheme.equalsIgnoreCase("asv") || scheme.equalsIgnoreCase("asvs") || scheme.equalsIgnoreCase("wasb") || scheme.equalsIgnoreCase("wasbs")); } /** * Puts in the authority of the default file system if it is a WASB file * system and the given URI's authority is null. * * @return The URI with reconstructed authority if necessary and possible. */ private static URI reconstructAuthorityIfNeeded(URI uri, Configuration conf) { if (null == uri.getAuthority()) { // If WASB is the default file system, get the authority from there URI defaultUri = FileSystem.getDefaultUri(conf); if (defaultUri != null && isWasbScheme(defaultUri.getScheme())) { try { // Reconstruct the URI with the authority from the default URI. return new URI(uri.getScheme(), defaultUri.getAuthority(), uri.getPath(), uri.getQuery(), uri.getFragment()); } catch (URISyntaxException e) { // This should never happen. throw new Error("Bad URI construction", e); } } } return uri; } @Override protected void checkPath(Path path) { // Make sure to reconstruct the path's authority if needed super.checkPath(new Path(reconstructAuthorityIfNeeded(path.toUri(), getConf()))); } @Override public void initialize(URI uri, Configuration conf) throws IOException, IllegalArgumentException { // Check authority for the URI to guarantee that it is non-null. uri = reconstructAuthorityIfNeeded(uri, conf); if (null == uri.getAuthority()) { final String errMsg = String .format("Cannot initialize WASB file system, URI authority not recognized."); throw new IllegalArgumentException(errMsg); } super.initialize(uri, conf); if (store == null) { store = createDefaultStore(conf); } instrumentation = new AzureFileSystemInstrumentation(conf); if (!conf.getBoolean(SKIP_AZURE_METRICS_PROPERTY_NAME, false)) { // Make sure the metrics system is available before interacting with Azure AzureFileSystemMetricsSystem.fileSystemStarted(); metricsSourceName = newMetricsSourceName(); String sourceDesc = "Azure Storage Volume File System metrics"; AzureFileSystemMetricsSystem.registerSource(metricsSourceName, sourceDesc, instrumentation); } store.initialize(uri, conf, instrumentation); setConf(conf); this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority()); this.workingDir = new Path("/user", UserGroupInformation.getCurrentUser().getShortUserName()) .makeQualified(getUri(), getWorkingDirectory()); this.blockSize = conf.getLong(AZURE_BLOCK_SIZE_PROPERTY_NAME, MAX_AZURE_BLOCK_SIZE); if (LOG.isDebugEnabled()) { LOG.debug("NativeAzureFileSystem. Initializing."); LOG.debug(" blockSize = " + conf.getLong(AZURE_BLOCK_SIZE_PROPERTY_NAME, MAX_AZURE_BLOCK_SIZE)); } } private NativeFileSystemStore createDefaultStore(Configuration conf) { actualStore = new AzureNativeFileSystemStore(); if (suppressRetryPolicy) { actualStore.suppressRetryPolicy(); } return actualStore; } /** * Azure Storage doesn't allow the blob names to end in a period, * so encode this here to work around that limitation. */ private static String encodeTrailingPeriod(String toEncode) { Matcher matcher = TRAILING_PERIOD_PATTERN.matcher(toEncode); return matcher.replaceAll(TRAILING_PERIOD_PLACEHOLDER); } /** * Reverse the encoding done by encodeTrailingPeriod(). */ private static String decodeTrailingPeriod(String toDecode) { Matcher matcher = TRAILING_PERIOD_PLACEHOLDER_PATTERN.matcher(toDecode); return matcher.replaceAll("."); } /** * Convert the path to a key. By convention, any leading or trailing slash is * removed, except for the special case of a single slash. */ @VisibleForTesting public String pathToKey(Path path) { // Convert the path to a URI to parse the scheme, the authority, and the // path from the path object. URI tmpUri = path.toUri(); String pathUri = tmpUri.getPath(); // The scheme and authority is valid. If the path does not exist add a "/" // separator to list the root of the container. Path newPath = path; if ("".equals(pathUri)) { newPath = new Path(tmpUri.toString() + Path.SEPARATOR); } // Verify path is absolute if the path refers to a windows drive scheme. if (!newPath.isAbsolute()) { throw new IllegalArgumentException("Path must be absolute: " + path); } String key = null; key = newPath.toUri().getPath(); key = removeTrailingSlash(key); key = encodeTrailingPeriod(key); if (key.length() == 1) { return key; } else { return key.substring(1); // remove initial slash } } // Remove any trailing slash except for the case of a single slash. private static String removeTrailingSlash(String key) { if (key.length() == 0 || key.length() == 1) { return key; } if (key.charAt(key.length() - 1) == '/') { return key.substring(0, key.length() - 1); } else { return key; } } private static Path keyToPath(String key) { if (key.equals("/")) { return new Path("/"); // container } return new Path("/" + decodeTrailingPeriod(key)); } /** * Get the absolute version of the path (fully qualified). * This is public for testing purposes. * * @param path * @return fully qualified path */ @VisibleForTesting public Path makeAbsolute(Path path) { if (path.isAbsolute()) { return path; } return new Path(workingDir, path); } /** * For unit test purposes, retrieves the AzureNativeFileSystemStore store * backing this file system. * * @return The store object. */ @VisibleForTesting public AzureNativeFileSystemStore getStore() { return actualStore; } NativeFileSystemStore getStoreInterface() { return store; } /** * Gets the metrics source for this file system. * This is mainly here for unit testing purposes. * * @return the metrics source. */ public AzureFileSystemInstrumentation getInstrumentation() { return instrumentation; } /** This optional operation is not yet supported. */ @Override public FSDataOutputStream append(Path f, int bufferSize, Progressable progress) throws IOException { throw new IOException("Not supported"); } @Override public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { return create(f, permission, overwrite, true, bufferSize, replication, blockSize, progress, (SelfRenewingLease) null); } /** * Get a self-renewing lease on the specified file. */ public SelfRenewingLease acquireLease(Path path) throws AzureException { String fullKey = pathToKey(makeAbsolute(path)); return getStore().acquireLease(fullKey); } @Override @SuppressWarnings("deprecation") public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { Path parent = f.getParent(); // Get exclusive access to folder if this is a directory designated // for atomic rename. The primary use case of for HBase write-ahead // log file management. SelfRenewingLease lease = null; if (store.isAtomicRenameKey(pathToKey(f))) { try { lease = acquireLease(parent); } catch (AzureException e) { String errorCode = ""; try { StorageException e2 = (StorageException) e.getCause(); errorCode = e2.getErrorCode(); } catch (Exception e3) { // do nothing if cast fails } if (errorCode.equals("BlobNotFound")) { throw new FileNotFoundException( "Cannot create file " + f.getName() + " because parent folder does not exist."); } LOG.warn("Got unexpected exception trying to get lease on " + pathToKey(parent) + ". " + e.getMessage()); throw e; } } // See if the parent folder exists. If not, throw error. // The exists() check will push any pending rename operation forward, // if there is one, and return false. // // At this point, we have exclusive access to the source folder // via the lease, so we will not conflict with an active folder // rename operation. if (!exists(parent)) { try { // This'll let the keep-alive thread exit as soon as it wakes up. lease.free(); } catch (Exception e) { LOG.warn("Unable to free lease because: " + e.getMessage()); } throw new FileNotFoundException( "Cannot create file " + f.getName() + " because parent folder does not exist."); } // Create file inside folder. FSDataOutputStream out = null; try { out = create(f, permission, overwrite, false, bufferSize, replication, blockSize, progress, lease); } finally { // Release exclusive access to folder. try { if (lease != null) { lease.free(); } } catch (Exception e) { IOUtils.cleanup(LOG, out); String msg = "Unable to free lease on " + parent.toUri(); LOG.error(msg); throw new IOException(msg, e); } } return out; } @Override @SuppressWarnings("deprecation") public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { // Check if file should be appended or overwritten. Assume that the file // is overwritten on if the CREATE and OVERWRITE create flags are set. Note // that any other combinations of create flags will result in an open new or // open with append. final EnumSet<CreateFlag> createflags = EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE); boolean overwrite = flags.containsAll(createflags); // Delegate the create non-recursive call. return this.createNonRecursive(f, permission, overwrite, bufferSize, replication, blockSize, progress); } @Override @SuppressWarnings("deprecation") public FSDataOutputStream createNonRecursive(Path f, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { return this.createNonRecursive(f, FsPermission.getFileDefault(), overwrite, bufferSize, replication, blockSize, progress); } /** * Create an Azure blob and return an output stream to use * to write data to it. * * @param f * @param permission * @param overwrite * @param createParent * @param bufferSize * @param replication * @param blockSize * @param progress * @param parentFolderLease Lease on parent folder (or null if * no lease). * @return * @throws IOException */ private FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, boolean createParent, int bufferSize, short replication, long blockSize, Progressable progress, SelfRenewingLease parentFolderLease) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("Creating file: " + f.toString()); } if (containsColon(f)) { throw new IOException("Cannot create file " + f + " through WASB that has colons in the name"); } Path absolutePath = makeAbsolute(f); String key = pathToKey(absolutePath); FileMetadata existingMetadata = store.retrieveMetadata(key); if (existingMetadata != null) { if (existingMetadata.isDir()) { throw new IOException("Cannot create file " + f + "; already exists as a directory."); } if (!overwrite) { throw new IOException("File already exists:" + f); } } Path parentFolder = absolutePath.getParent(); if (parentFolder != null && parentFolder.getParent() != null) { // skip root // Update the parent folder last modified time if the parent folder // already exists. String parentKey = pathToKey(parentFolder); FileMetadata parentMetadata = store.retrieveMetadata(parentKey); if (parentMetadata != null && parentMetadata.isDir() && parentMetadata.getBlobMaterialization() == BlobMaterialization.Explicit) { if (parentFolderLease != null) { store.updateFolderLastModifiedTime(parentKey, parentFolderLease); } else { updateParentFolderLastModifiedTime(key); } } else { // Make sure that the parent folder exists. // Create it using inherited permissions from the first existing directory going up the path Path firstExisting = parentFolder.getParent(); FileMetadata metadata = store.retrieveMetadata(pathToKey(firstExisting)); while (metadata == null) { // Guaranteed to terminate properly because we will eventually hit root, which will return non-null metadata firstExisting = firstExisting.getParent(); metadata = store.retrieveMetadata(pathToKey(firstExisting)); } mkdirs(parentFolder, metadata.getPermissionStatus().getPermission(), true); } } // Mask the permission first (with the default permission mask as well). FsPermission masked = applyUMask(permission, UMaskApplyMode.NewFile); PermissionStatus permissionStatus = createPermissionStatus(masked); OutputStream bufOutStream; if (store.isPageBlobKey(key)) { // Store page blobs directly in-place without renames. bufOutStream = store.storefile(key, permissionStatus); } else { // This is a block blob, so open the output blob stream based on the // encoded key. // String keyEncoded = encodeKey(key); // First create a blob at the real key, pointing back to the temporary file // This accomplishes a few things: // 1. Makes sure we can create a file there. // 2. Makes it visible to other concurrent threads/processes/nodes what // we're // doing. // 3. Makes it easier to restore/cleanup data in the event of us crashing. store.storeEmptyLinkFile(key, keyEncoded, permissionStatus); // The key is encoded to point to a common container at the storage server. // This reduces the number of splits on the server side when load balancing. // Ingress to Azure storage can take advantage of earlier splits. We remove // the root path to the key and prefix a random GUID to the tail (or leaf // filename) of the key. Keys are thus broadly and randomly distributed over // a single container to ease load balancing on the storage server. When the // blob is committed it is renamed to its earlier key. Uncommitted blocks // are not cleaned up and we leave it to Azure storage to garbage collect // these // blocks. bufOutStream = new NativeAzureFsOutputStream(store.storefile(keyEncoded, permissionStatus), key, keyEncoded); } // Construct the data output stream from the buffered output stream. FSDataOutputStream fsOut = new FSDataOutputStream(bufOutStream, statistics); // Increment the counter instrumentation.fileCreated(); // Return data output stream to caller. return fsOut; } @Override @Deprecated public boolean delete(Path path) throws IOException { return delete(path, true); } @Override public boolean delete(Path f, boolean recursive) throws IOException { return delete(f, recursive, false); } /** * Delete the specified file or folder. The parameter * skipParentFolderLastModifidedTimeUpdate * is used in the case of atomic folder rename redo. In that case, there is * a lease on the parent folder, so (without reworking the code) modifying * the parent folder update time will fail because of a conflict with the * lease. Since we are going to delete the folder soon anyway so accurate * modified time is not necessary, it's easier to just skip * the modified time update. * * @param f * @param recursive * @param skipParentFolderLastModifidedTimeUpdate If true, don't update the folder last * modified time. * @return true if and only if the file is deleted * @throws IOException */ public boolean delete(Path f, boolean recursive, boolean skipParentFolderLastModifidedTimeUpdate) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("Deleting file: " + f.toString()); } Path absolutePath = makeAbsolute(f); String key = pathToKey(absolutePath); // Capture the metadata for the path. // FileMetadata metaFile = store.retrieveMetadata(key); if (null == metaFile) { // The path to be deleted does not exist. return false; } // The path exists, determine if it is a folder containing objects, // an empty folder, or a simple file and take the appropriate actions. if (!metaFile.isDir()) { // The path specifies a file. We need to check the parent path // to make sure it's a proper materialized directory before we // delete the file. Otherwise we may get into a situation where // the file we were deleting was the last one in an implicit directory // (e.g. the blob store only contains the blob a/b and there's no // corresponding directory blob a) and that would implicitly delete // the directory as well, which is not correct. Path parentPath = absolutePath.getParent(); if (parentPath.getParent() != null) {// Not root String parentKey = pathToKey(parentPath); FileMetadata parentMetadata = store.retrieveMetadata(parentKey); if (!parentMetadata.isDir()) { // Invalid state: the parent path is actually a file. Throw. throw new AzureException("File " + f + " has a parent directory " + parentPath + " which is also a file. Can't resolve."); } if (parentMetadata.getBlobMaterialization() == BlobMaterialization.Implicit) { if (LOG.isDebugEnabled()) { LOG.debug("Found an implicit parent directory while trying to" + " delete the file " + f + ". Creating the directory blob for" + " it in " + parentKey + "."); } store.storeEmptyFolder(parentKey, createPermissionStatus(FsPermission.getDefault())); } else { if (!skipParentFolderLastModifidedTimeUpdate) { updateParentFolderLastModifiedTime(key); } } } store.delete(key); instrumentation.fileDeleted(); } else { // The path specifies a folder. Recursively delete all entries under the // folder. Path parentPath = absolutePath.getParent(); if (parentPath.getParent() != null) { String parentKey = pathToKey(parentPath); FileMetadata parentMetadata = store.retrieveMetadata(parentKey); if (parentMetadata.getBlobMaterialization() == BlobMaterialization.Implicit) { if (LOG.isDebugEnabled()) { LOG.debug("Found an implicit parent directory while trying to" + " delete the directory " + f + ". Creating the directory blob for" + " it in " + parentKey + "."); } store.storeEmptyFolder(parentKey, createPermissionStatus(FsPermission.getDefault())); } } // List all the blobs in the current folder. String priorLastKey = null; PartialListing listing = store.listAll(key, AZURE_LIST_ALL, 1, priorLastKey); FileMetadata[] contents = listing.getFiles(); if (!recursive && contents.length > 0) { // The folder is non-empty and recursive delete was not specified. // Throw an exception indicating that a non-recursive delete was // specified for a non-empty folder. throw new IOException("Non-recursive delete of non-empty directory " + f.toString()); } // Delete all the files in the folder. for (FileMetadata p : contents) { // Tag on the directory name found as the suffix of the suffix of the // parent directory to get the new absolute path. String suffix = p.getKey().substring(p.getKey().lastIndexOf(PATH_DELIMITER)); if (!p.isDir()) { store.delete(key + suffix); instrumentation.fileDeleted(); } else { // Recursively delete contents of the sub-folders. Notice this also // deletes the blob for the directory. if (!delete(new Path(f.toString() + suffix), true)) { return false; } } } store.delete(key); // Update parent directory last modified time Path parent = absolutePath.getParent(); if (parent != null && parent.getParent() != null) { // not root if (!skipParentFolderLastModifidedTimeUpdate) { updateParentFolderLastModifiedTime(key); } } instrumentation.directoryDeleted(); } // File or directory was successfully deleted. return true; } @Override public FileStatus getFileStatus(Path f) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("Getting the file status for " + f.toString()); } // Capture the absolute path and the path to key. Path absolutePath = makeAbsolute(f); String key = pathToKey(absolutePath); if (key.length() == 0) { // root always exists return newDirectory(null, absolutePath); } // The path is either a folder or a file. Retrieve metadata to // determine if it is a directory or file. FileMetadata meta = store.retrieveMetadata(key); if (meta != null) { if (meta.isDir()) { // The path is a folder with files in it. // if (LOG.isDebugEnabled()) { LOG.debug("Path " + f.toString() + "is a folder."); } // If a rename operation for the folder was pending, redo it. // Then the file does not exist, so signal that. if (conditionalRedoFolderRename(f)) { throw new FileNotFoundException(absolutePath + ": No such file or directory."); } // Return reference to the directory object. return newDirectory(meta, absolutePath); } // The path is a file. if (LOG.isDebugEnabled()) { LOG.debug("Found the path: " + f.toString() + " as a file."); } // Return with reference to a file object. return newFile(meta, absolutePath); } // File not found. Throw exception no such file or directory. // throw new FileNotFoundException(absolutePath + ": No such file or directory."); } // Return true if there is a rename pending and we redo it, otherwise false. private boolean conditionalRedoFolderRename(Path f) throws IOException { // Can't rename /, so return immediately in that case. if (f.getName().equals("")) { return false; } // Check if there is a -RenamePending.json file for this folder, and if so, // redo the rename. Path absoluteRenamePendingFile = renamePendingFilePath(f); if (exists(absoluteRenamePendingFile)) { FolderRenamePending pending = new FolderRenamePending(absoluteRenamePendingFile, this); pending.redo(); return true; } else { return false; } } // Return the path name that would be used for rename of folder with path f. private Path renamePendingFilePath(Path f) { Path absPath = makeAbsolute(f); String key = pathToKey(absPath); key += "-RenamePending.json"; return keyToPath(key); } @Override public URI getUri() { return uri; } /** * Retrieve the status of a given path if it is a file, or of all the * contained files if it is a directory. */ @Override public FileStatus[] listStatus(Path f) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("Listing status for " + f.toString()); } Path absolutePath = makeAbsolute(f); String key = pathToKey(absolutePath); Set<FileStatus> status = new TreeSet<FileStatus>(); FileMetadata meta = store.retrieveMetadata(key); if (meta != null) { if (!meta.isDir()) { if (LOG.isDebugEnabled()) { LOG.debug("Found path as a file"); } return new FileStatus[] { newFile(meta, absolutePath) }; } String partialKey = null; PartialListing listing = store.list(key, AZURE_LIST_ALL, 1, partialKey); // For any -RenamePending.json files in the listing, // push the rename forward. boolean renamed = conditionalRedoFolderRenames(listing); // If any renames were redone, get another listing, // since the current one may have changed due to the redo. if (renamed) { listing = store.list(key, AZURE_LIST_ALL, 1, partialKey); } for (FileMetadata fileMetadata : listing.getFiles()) { Path subpath = keyToPath(fileMetadata.getKey()); // Test whether the metadata represents a file or directory and // add the appropriate metadata object. // // Note: There was a very old bug here where directories were added // to the status set as files flattening out recursive listings // using "-lsr" down the file system hierarchy. if (fileMetadata.isDir()) { // Make sure we hide the temp upload folder if (fileMetadata.getKey().equals(AZURE_TEMP_FOLDER)) { // Don't expose that. continue; } status.add(newDirectory(fileMetadata, subpath)); } else { status.add(newFile(fileMetadata, subpath)); } } if (LOG.isDebugEnabled()) { LOG.debug("Found path as a directory with " + status.size() + " files in it."); } } else { // There is no metadata found for the path. if (LOG.isDebugEnabled()) { LOG.debug("Did not find any metadata for path: " + key); } throw new FileNotFoundException("File" + f + " does not exist."); } return status.toArray(new FileStatus[0]); } // Redo any folder renames needed if there are rename pending files in the // directory listing. Return true if one or more redo operations were done. private boolean conditionalRedoFolderRenames(PartialListing listing) throws IllegalArgumentException, IOException { boolean renamed = false; for (FileMetadata fileMetadata : listing.getFiles()) { Path subpath = keyToPath(fileMetadata.getKey()); if (isRenamePendingFile(subpath)) { FolderRenamePending pending = new FolderRenamePending(subpath, this); pending.redo(); renamed = true; } } return renamed; } // True if this is a folder rename pending file, else false. private boolean isRenamePendingFile(Path path) { return path.toString().endsWith(FolderRenamePending.SUFFIX); } private FileStatus newFile(FileMetadata meta, Path path) { return new FileStatus(meta.getLength(), false, 1, blockSize, meta.getLastModified(), 0, meta.getPermissionStatus().getPermission(), meta.getPermissionStatus().getUserName(), meta.getPermissionStatus().getGroupName(), path.makeQualified(getUri(), getWorkingDirectory())); } private FileStatus newDirectory(FileMetadata meta, Path path) { return new FileStatus(0, true, 1, blockSize, meta == null ? 0 : meta.getLastModified(), 0, meta == null ? FsPermission.getDefault() : meta.getPermissionStatus().getPermission(), meta == null ? "" : meta.getPermissionStatus().getUserName(), meta == null ? "" : meta.getPermissionStatus().getGroupName(), path.makeQualified(getUri(), getWorkingDirectory())); } private static enum UMaskApplyMode { NewFile, NewDirectory, NewDirectoryNoUmask, ChangeExistingFile, ChangeExistingDirectory, } /** * Applies the applicable UMASK's on the given permission. * * @param permission * The permission to mask. * @param applyMode * Whether to also apply the default umask. * @return The masked persmission. */ private FsPermission applyUMask(final FsPermission permission, final UMaskApplyMode applyMode) { FsPermission newPermission = new FsPermission(permission); // Apply the default umask - this applies for new files or directories. if (applyMode == UMaskApplyMode.NewFile || applyMode == UMaskApplyMode.NewDirectory) { newPermission = newPermission.applyUMask(FsPermission.getUMask(getConf())); } return newPermission; } /** * Creates the PermissionStatus object to use for the given permission, based * on the current user in context. * * @param permission * The permission for the file. * @return The permission status object to use. * @throws IOException * If login fails in getCurrentUser */ private PermissionStatus createPermissionStatus(FsPermission permission) throws IOException { // Create the permission status for this file based on current user return new PermissionStatus(UserGroupInformation.getCurrentUser().getShortUserName(), getConf().get(AZURE_DEFAULT_GROUP_PROPERTY_NAME, AZURE_DEFAULT_GROUP_DEFAULT), permission); } @Override public boolean mkdirs(Path f, FsPermission permission) throws IOException { return mkdirs(f, permission, false); } public boolean mkdirs(Path f, FsPermission permission, boolean noUmask) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("Creating directory: " + f.toString()); } if (containsColon(f)) { throw new IOException("Cannot create directory " + f + " through WASB that has colons in the name"); } Path absolutePath = makeAbsolute(f); PermissionStatus permissionStatus = null; if (noUmask) { // ensure owner still has wx permissions at the minimum permissionStatus = createPermissionStatus( applyUMask(FsPermission.createImmutable((short) (permission.toShort() | USER_WX_PERMISION)), UMaskApplyMode.NewDirectoryNoUmask)); } else { permissionStatus = createPermissionStatus(applyUMask(permission, UMaskApplyMode.NewDirectory)); } ArrayList<String> keysToCreateAsFolder = new ArrayList<String>(); ArrayList<String> keysToUpdateAsFolder = new ArrayList<String>(); boolean childCreated = false; // Check that there is no file in the parent chain of the given path. for (Path current = absolutePath, parent = current.getParent(); parent != null; // Stop when you get to the root current = parent, parent = current.getParent()) { String currentKey = pathToKey(current); FileMetadata currentMetadata = store.retrieveMetadata(currentKey); if (currentMetadata != null && !currentMetadata.isDir()) { throw new IOException( "Cannot create directory " + f + " because " + current + " is an existing file."); } else if (currentMetadata == null) { keysToCreateAsFolder.add(currentKey); childCreated = true; } else { // The directory already exists. Its last modified time need to be // updated if there is a child directory created under it. if (childCreated) { keysToUpdateAsFolder.add(currentKey); } childCreated = false; } } for (String currentKey : keysToCreateAsFolder) { store.storeEmptyFolder(currentKey, permissionStatus); } instrumentation.directoryCreated(); // otherwise throws exception return true; } @Override public FSDataInputStream open(Path f, int bufferSize) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("Opening file: " + f.toString()); } Path absolutePath = makeAbsolute(f); String key = pathToKey(absolutePath); FileMetadata meta = store.retrieveMetadata(key); if (meta == null) { throw new FileNotFoundException(f.toString()); } if (meta.isDir()) { throw new FileNotFoundException(f.toString() + " is a directory not a file."); } return new FSDataInputStream(new BufferedFSInputStream( new NativeAzureFsInputStream(store.retrieve(key), key, meta.getLength()), bufferSize)); } @Override public boolean rename(Path src, Path dst) throws IOException { FolderRenamePending renamePending = null; if (LOG.isDebugEnabled()) { LOG.debug("Moving " + src + " to " + dst); } if (containsColon(dst)) { throw new IOException("Cannot rename to file " + dst + " through WASB that has colons in the name"); } String srcKey = pathToKey(makeAbsolute(src)); if (srcKey.length() == 0) { // Cannot rename root of file system return false; } // Figure out the final destination Path absoluteDst = makeAbsolute(dst); String dstKey = pathToKey(absoluteDst); FileMetadata dstMetadata = store.retrieveMetadata(dstKey); if (dstMetadata != null && dstMetadata.isDir()) { // It's an existing directory. dstKey = pathToKey(makeAbsolute(new Path(dst, src.getName()))); if (LOG.isDebugEnabled()) { LOG.debug("Destination " + dst + " is a directory, adjusted the destination to be " + dstKey); } } else if (dstMetadata != null) { // Attempting to overwrite a file using rename() if (LOG.isDebugEnabled()) { LOG.debug("Destination " + dst + " is an already existing file, failing the rename."); } return false; } else { // Check that the parent directory exists. FileMetadata parentOfDestMetadata = store.retrieveMetadata(pathToKey(absoluteDst.getParent())); if (parentOfDestMetadata == null) { if (LOG.isDebugEnabled()) { LOG.debug("Parent of the destination " + dst + " doesn't exist, failing the rename."); } return false; } else if (!parentOfDestMetadata.isDir()) { if (LOG.isDebugEnabled()) { LOG.debug("Parent of the destination " + dst + " is a file, failing the rename."); } return false; } } FileMetadata srcMetadata = store.retrieveMetadata(srcKey); if (srcMetadata == null) { // Source doesn't exist if (LOG.isDebugEnabled()) { LOG.debug("Source " + src + " doesn't exist, failing the rename."); } return false; } else if (!srcMetadata.isDir()) { if (LOG.isDebugEnabled()) { LOG.debug("Source " + src + " found as a file, renaming."); } store.rename(srcKey, dstKey); } else { // Prepare for, execute and clean up after of all files in folder, and // the root file, and update the last modified time of the source and // target parent folders. The operation can be redone if it fails part // way through, by applying the "Rename Pending" file. // The following code (internally) only does atomic rename preparation // and lease management for page blob folders, limiting the scope of the // operation to HBase log file folders, where atomic rename is required. // In the future, we could generalize it easily to all folders. renamePending = prepareAtomicFolderRename(srcKey, dstKey); renamePending.execute(); if (LOG.isDebugEnabled()) { LOG.debug("Renamed " + src + " to " + dst + " successfully."); } renamePending.cleanup(); return true; } // Update the last-modified time of the parent folders of both source // and destination. updateParentFolderLastModifiedTime(srcKey); updateParentFolderLastModifiedTime(dstKey); if (LOG.isDebugEnabled()) { LOG.debug("Renamed " + src + " to " + dst + " successfully."); } return true; } /** * Update the last-modified time of the parent folder of the file * identified by key. * @param key * @throws IOException */ private void updateParentFolderLastModifiedTime(String key) throws IOException { Path parent = makeAbsolute(keyToPath(key)).getParent(); if (parent != null && parent.getParent() != null) { // not root String parentKey = pathToKey(parent); // ensure the parent is a materialized folder FileMetadata parentMetadata = store.retrieveMetadata(parentKey); // The metadata could be null if the implicit folder only contains a // single file. In this case, the parent folder no longer exists if the // file is renamed; so we can safely ignore the null pointer case. if (parentMetadata != null) { if (parentMetadata.isDir() && parentMetadata.getBlobMaterialization() == BlobMaterialization.Implicit) { store.storeEmptyFolder(parentKey, createPermissionStatus(FsPermission.getDefault())); } if (store.isAtomicRenameKey(parentKey)) { SelfRenewingLease lease = null; try { lease = leaseSourceFolder(parentKey); store.updateFolderLastModifiedTime(parentKey, lease); } catch (AzureException e) { String errorCode = ""; try { StorageException e2 = (StorageException) e.getCause(); errorCode = e2.getErrorCode(); } catch (Exception e3) { // do nothing if cast fails } if (errorCode.equals("BlobNotFound")) { throw new FileNotFoundException("Folder does not exist: " + parentKey); } LOG.warn("Got unexpected exception trying to get lease on " + parentKey + ". " + e.getMessage()); throw e; } finally { try { if (lease != null) { lease.free(); } } catch (Exception e) { LOG.error("Unable to free lease on " + parentKey, e); } } } else { store.updateFolderLastModifiedTime(parentKey, null); } } } } /** * If the source is a page blob folder, * prepare to rename this folder atomically. This means to get exclusive * access to the source folder, and record the actions to be performed for * this rename in a "Rename Pending" file. This code was designed to * meet the needs of HBase, which requires atomic rename of write-ahead log * (WAL) folders for correctness. * * Before calling this method, the caller must ensure that the source is a * folder. * * For non-page-blob directories, prepare the in-memory information needed, * but don't take the lease or write the redo file. This is done to limit the * scope of atomic folder rename to HBase, at least at the time of writing * this code. * * @param srcKey Source folder name. * @param dstKey Destination folder name. * @throws IOException */ private FolderRenamePending prepareAtomicFolderRename(String srcKey, String dstKey) throws IOException { if (store.isAtomicRenameKey(srcKey)) { // Block unwanted concurrent access to source folder. SelfRenewingLease lease = leaseSourceFolder(srcKey); // Prepare in-memory information needed to do or redo a folder rename. FolderRenamePending renamePending = new FolderRenamePending(srcKey, dstKey, lease, this); // Save it to persistent storage to help recover if the operation fails. renamePending.writeFile(this); return renamePending; } else { FolderRenamePending renamePending = new FolderRenamePending(srcKey, dstKey, null, this); return renamePending; } } /** * Get a self-renewing Azure blob lease on the source folder zero-byte file. */ private SelfRenewingLease leaseSourceFolder(String srcKey) throws AzureException { return store.acquireLease(srcKey); } /** * Return an array containing hostnames, offset and size of * portions of the given file. For WASB we'll just lie and give * fake hosts to make sure we get many splits in MR jobs. */ @Override public BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len) throws IOException { if (file == null) { return null; } if ((start < 0) || (len < 0)) { throw new IllegalArgumentException("Invalid start or len parameter"); } if (file.getLen() < start) { return new BlockLocation[0]; } final String blobLocationHost = getConf().get(AZURE_BLOCK_LOCATION_HOST_PROPERTY_NAME, AZURE_BLOCK_LOCATION_HOST_DEFAULT); final String[] name = { blobLocationHost }; final String[] host = { blobLocationHost }; long blockSize = file.getBlockSize(); if (blockSize <= 0) { throw new IllegalArgumentException( "The block size for the given file is not a positive number: " + blockSize); } int numberOfLocations = (int) (len / blockSize) + ((len % blockSize == 0) ? 0 : 1); BlockLocation[] locations = new BlockLocation[numberOfLocations]; for (int i = 0; i < locations.length; i++) { long currentOffset = start + (i * blockSize); long currentLength = Math.min(blockSize, start + len - currentOffset); locations[i] = new BlockLocation(name, host, currentOffset, currentLength); } return locations; } /** * Set the working directory to the given directory. */ @Override public void setWorkingDirectory(Path newDir) { workingDir = makeAbsolute(newDir); } @Override public Path getWorkingDirectory() { return workingDir; } @Override public void setPermission(Path p, FsPermission permission) throws IOException { Path absolutePath = makeAbsolute(p); String key = pathToKey(absolutePath); FileMetadata metadata = store.retrieveMetadata(key); if (metadata == null) { throw new FileNotFoundException("File doesn't exist: " + p); } permission = applyUMask(permission, metadata.isDir() ? UMaskApplyMode.ChangeExistingDirectory : UMaskApplyMode.ChangeExistingFile); if (metadata.getBlobMaterialization() == BlobMaterialization.Implicit) { // It's an implicit folder, need to materialize it. store.storeEmptyFolder(key, createPermissionStatus(permission)); } else if (!metadata.getPermissionStatus().getPermission().equals(permission)) { store.changePermissionStatus(key, new PermissionStatus(metadata.getPermissionStatus().getUserName(), metadata.getPermissionStatus().getGroupName(), permission)); } } @Override public void setOwner(Path p, String username, String groupname) throws IOException { Path absolutePath = makeAbsolute(p); String key = pathToKey(absolutePath); FileMetadata metadata = store.retrieveMetadata(key); if (metadata == null) { throw new FileNotFoundException("File doesn't exist: " + p); } PermissionStatus newPermissionStatus = new PermissionStatus( username == null ? metadata.getPermissionStatus().getUserName() : username, groupname == null ? metadata.getPermissionStatus().getGroupName() : groupname, metadata.getPermissionStatus().getPermission()); if (metadata.getBlobMaterialization() == BlobMaterialization.Implicit) { // It's an implicit folder, need to materialize it. store.storeEmptyFolder(key, newPermissionStatus); } else { store.changePermissionStatus(key, newPermissionStatus); } } @Override public synchronized void close() throws IOException { if (isClosed) { return; } // Call the base close() to close any resources there. super.close(); // Close the store to close any resources there - e.g. the bandwidth // updater thread would be stopped at this time. store.close(); // Notify the metrics system that this file system is closed, which may // trigger one final metrics push to get the accurate final file system // metrics out. long startTime = System.currentTimeMillis(); if (!getConf().getBoolean(SKIP_AZURE_METRICS_PROPERTY_NAME, false)) { AzureFileSystemMetricsSystem.unregisterSource(metricsSourceName); AzureFileSystemMetricsSystem.fileSystemClosed(); } if (LOG.isDebugEnabled()) { LOG.debug("Submitting metrics when file system closed took " + (System.currentTimeMillis() - startTime) + " ms."); } isClosed = true; } /** * A handler that defines what to do with blobs whose upload was * interrupted. */ private abstract class DanglingFileHandler { abstract void handleFile(FileMetadata file, FileMetadata tempFile) throws IOException; } /** * Handler implementation for just deleting dangling files and cleaning * them up. */ private class DanglingFileDeleter extends DanglingFileHandler { @Override void handleFile(FileMetadata file, FileMetadata tempFile) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("Deleting dangling file " + file.getKey()); } store.delete(file.getKey()); store.delete(tempFile.getKey()); } } /** * Handler implementation for just moving dangling files to recovery * location (/lost+found). */ private class DanglingFileRecoverer extends DanglingFileHandler { private final Path destination; DanglingFileRecoverer(Path destination) { this.destination = destination; } @Override void handleFile(FileMetadata file, FileMetadata tempFile) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("Recovering " + file.getKey()); } // Move to the final destination String finalDestinationKey = pathToKey(new Path(destination, file.getKey())); store.rename(tempFile.getKey(), finalDestinationKey); if (!finalDestinationKey.equals(file.getKey())) { // Delete the empty link file now that we've restored it. store.delete(file.getKey()); } } } /** * Check if a path has colons in its name */ private boolean containsColon(Path p) { return p.toUri().getPath().toString().contains(":"); } /** * Implements recover and delete (-move and -delete) behaviors for handling * dangling files (blobs whose upload was interrupted). * * @param root * The root path to check from. * @param handler * The handler that deals with dangling files. */ private void handleFilesWithDanglingTempData(Path root, DanglingFileHandler handler) throws IOException { // Calculate the cut-off for when to consider a blob to be dangling. long cutoffForDangling = new Date().getTime() - getConf().getInt(AZURE_TEMP_EXPIRY_PROPERTY_NAME, AZURE_TEMP_EXPIRY_DEFAULT) * 1000; // Go over all the blobs under the given root and look for blobs to // recover. String priorLastKey = null; do { PartialListing listing = store.listAll(pathToKey(root), AZURE_LIST_ALL, AZURE_UNBOUNDED_DEPTH, priorLastKey); for (FileMetadata file : listing.getFiles()) { if (!file.isDir()) { // We don't recover directory blobs // See if this blob has a link in it (meaning it's a place-holder // blob for when the upload to the temp blob is complete). String link = store.getLinkInFileMetadata(file.getKey()); if (link != null) { // It has a link, see if the temp blob it is pointing to is // existent and old enough to be considered dangling. FileMetadata linkMetadata = store.retrieveMetadata(link); if (linkMetadata != null && linkMetadata.getLastModified() >= cutoffForDangling) { // Found one! handler.handleFile(file, linkMetadata); } } } } priorLastKey = listing.getPriorLastKey(); } while (priorLastKey != null); } /** * Looks under the given root path for any blob that are left "dangling", * meaning that they are place-holder blobs that we created while we upload * the data to a temporary blob, but for some reason we crashed in the middle * of the upload and left them there. If any are found, we move them to the * destination given. * * @param root * The root path to consider. * @param destination * The destination path to move any recovered files to. * @throws IOException */ public void recoverFilesWithDanglingTempData(Path root, Path destination) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("Recovering files with dangling temp data in " + root); } handleFilesWithDanglingTempData(root, new DanglingFileRecoverer(destination)); } /** * Looks under the given root path for any blob that are left "dangling", * meaning that they are place-holder blobs that we created while we upload * the data to a temporary blob, but for some reason we crashed in the middle * of the upload and left them there. If any are found, we delete them. * * @param root * The root path to consider. * @throws IOException */ public void deleteFilesWithDanglingTempData(Path root) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("Deleting files with dangling temp data in " + root); } handleFilesWithDanglingTempData(root, new DanglingFileDeleter()); } @Override protected void finalize() throws Throwable { LOG.debug("finalize() called."); close(); super.finalize(); } /** * Encode the key with a random prefix for load balancing in Azure storage. * Upload data to a random temporary file then do storage side renaming to * recover the original key. * * @param aKey * @return Encoded version of the original key. */ private static String encodeKey(String aKey) { // Get the tail end of the key name. // String fileName = aKey.substring(aKey.lastIndexOf(Path.SEPARATOR) + 1, aKey.length()); // Construct the randomized prefix of the file name. The prefix ensures the // file always drops into the same folder but with a varying tail key name. String filePrefix = AZURE_TEMP_FOLDER + Path.SEPARATOR + UUID.randomUUID().toString(); // Concatenate the randomized prefix with the tail of the key name. String randomizedKey = filePrefix + fileName; // Return to the caller with the randomized key. return randomizedKey; } }