Java tutorial
/* * Copyright (c) 2013, the authors. * * This file is part of 'DXFS'. * * DXFS is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * DXFS is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with DXFS. If not, see <http://www.gnu.org/licenses/>. */ package nextflow.fs.dx; import java.io.BufferedReader; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.URI; import java.nio.channels.FileChannel; import java.nio.channels.SeekableByteChannel; import java.nio.charset.Charset; import java.nio.file.AccessMode; import java.nio.file.CopyOption; import java.nio.file.DirectoryStream; import java.nio.file.DirectoryStream.Filter; import java.nio.file.FileAlreadyExistsException; import java.nio.file.FileStore; import java.nio.file.FileSystem; import java.nio.file.FileSystemAlreadyExistsException; import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.OpenOption; import java.nio.file.Path; import java.nio.file.ProviderMismatchException; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.FileAttribute; import java.nio.file.attribute.FileAttributeView; import java.nio.file.spi.FileSystemProvider; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; import com.fasterxml.jackson.databind.JsonNode; import nextflow.fs.dx.api.DxApi; import nextflow.fs.dx.api.DxHttpClient; import nextflow.fs.dx.api.DxJson; import org.apache.http.HttpVersion; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.params.ClientPNames; import org.apache.http.client.params.CookiePolicy; import org.apache.http.params.CoreProtocolPNames; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * NIO2 File system provider for DnaNexus cloud storage * * @author Paolo Di Tommaso <paolo.ditommaso@gmail.com> * @author Beatriz San Juan <bmsanjuan@gmail.com> * */ public class DxFileSystemProvider extends FileSystemProvider { private static final Logger log = LoggerFactory.getLogger(DxFileSystemProvider.class); /** * The *scheme* defined by this provider */ public static String SCHEME = "dxfs"; /** * Pattern matching a DnaNexus URI e.g. dxfs://container:some/path */ public static Pattern URI_PATTERN = Pattern.compile("^" + SCHEME + "://(([^/]+):)?(.*)$"); /** * Pattern matching a DnaNexus project or context ID */ static Pattern CONTEXT_ID_PATTERN = Pattern.compile("^(project-|container-)[a-zA-Z0-9]{24}$"); /** * Pattern matching a DnaNexus file ID */ public static Pattern FILE_ID_PATTERN = Pattern.compile("file-[a-zA-Z0-9]{24}"); /** * Hold DnaNexus context ID, either a project id or a data container container * <p> * Read more * https://wiki.dnanexus.com/API-Specification-v1.0.0/Data-Containers# */ final String defaultContextId; final ConcurrentHashMap<String, DxFileSystem> fileSystems = new ConcurrentHashMap<>(); final DxApi api; public DxFileSystemProvider() { this.defaultContextId = getDefaultDxContextId(); this.api = DxApi.getInstance(); } protected DxFileSystemProvider(String containerId, DxApi api) { this.defaultContextId = containerId; this.api = api; } /** * Find out the default DnaNexus project id in the default user configuration file, * i.e. the file {@code $HOME/.dnanexus_config/environment.json} * * @return The string value */ static String getDefaultDxContextId() { String result; @SuppressWarnings({ "unchecked", "rawtypes" }) Map<String, String> props = new HashMap(System.getProperties()); result = getContextIdByMap(props, null); if (result == null) { result = getContextIdByMap(System.getenv(), null); } if (result == null) { String home = System.getProperty("user.home"); File config = new File(home, ".dnanexus_config/environment.json"); if (!config.exists()) { return null; } result = getContextIdByConfig(config); log.debug("Using DX_PROJECT_CONTEXT_ID = {} in config file: {}", result, config); } return result; } static String getContextIdByMap(Map<String, ?> map, String defValue) { if (map.containsKey("DX_WORKSPACE_ID")) { String result = map.get("DX_WORKSPACE_ID").toString(); log.debug("Using DX_WORKSPACE_ID = {}", result); return result; } else if (map.containsKey("DX_PROJECT_CONTEXT_ID")) { String result = map.get("DX_PROJECT_CONTEXT_ID").toString(); log.debug("Using DX_PROJECT_CONTEXT_ID = {}", result); return result; } if (defValue != null) { log.debug("Using default context id = {}", defValue); } return defValue; } /** * Find out the default DnaNexus project id in the specified configuration file * * @return The string value */ static String getContextIdByConfig(File config) { StringBuilder buffer = new StringBuilder(); try { BufferedReader reader = Files.newBufferedReader(config.toPath(), Charset.defaultCharset()); String line; while ((line = reader.readLine()) != null) { buffer.append(line).append('\n'); } JsonNode object = DxJson.parseJson(buffer.toString()); return object.get("DX_PROJECT_CONTEXT_ID").textValue(); } catch (FileNotFoundException e) { throw new IllegalStateException(String.format( "Unable to load DnaNexus configuration file: %s -- cannot configure file system", config), e); } catch (IOException e) { throw new IllegalStateException("Unable to configure DnaNexus file system", e); } } protected DxApi api() { return api; } @Override public String getScheme() { return SCHEME; } @Override public Path getPath(URI uri) { log.trace("Get path by URI: {}", uri); PathTokens tokens = resolveUri(uri, defaultContextId); DxFileSystem dxFileSystem = getOrCreateFileSystem(tokens.contextId, tokens.name); return new DxPath(dxFileSystem, tokens.filePath); } protected DxFileSystem newFileSystem() { return newFileSystem(defaultContextId, defaultContextId); } protected DxFileSystem newFileSystem(String contextId, String label) { if (contextId == null) { throw new IllegalStateException("Missing 'contextId' attribute"); } return new DxFileSystem(this, contextId, label); } @Override public final FileSystem newFileSystem(URI uri, Map<String, ?> env) { final String defContextId = getContextIdByMap(env, defaultContextId); final PathTokens tokens = resolveUri(uri, defContextId); final String dxContextId = tokens.contextId; final DxFileSystem result = newFileSystem(dxContextId, tokens.name); if (fileSystems.putIfAbsent(dxContextId, result) != null) { throw new FileSystemAlreadyExistsException(); } return result; } // -- package private final DxFileSystem getOrCreateFileSystem(String contextId, String name) { DxFileSystem dxFileSystem = fileSystems.get(contextId); if (dxFileSystem == null) { log.debug("Creating a new DxFileSystem object with context-id: {}", contextId); dxFileSystem = newFileSystem(contextId, name); DxFileSystem former = fileSystems.putIfAbsent(contextId, dxFileSystem); if (former != null) { log.trace( "Look ma, got a concurrent creation of a DxFileSystem for context-id: {} -- using the previous instance", contextId); return former; } } return dxFileSystem; } // -- package private final DxFileSystem getOrCreateFileSystem(String contextId) { return getOrCreateFileSystem(contextId, contextId); } static class PathTokens { /** Descriptive name of the context/project */ String name; /** The real container/project id */ String contextId; /** The fil path in the container/project */ String filePath; @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; PathTokens that = (PathTokens) o; if (contextId != null ? !contextId.equals(that.contextId) : that.contextId != null) return false; if (filePath != null ? !filePath.equals(that.filePath) : that.filePath != null) return false; if (name != null ? !name.equals(that.name) : that.name != null) return false; return true; } @Override public int hashCode() { int result = name != null ? name.hashCode() : 0; result = 31 * result + (contextId != null ? contextId.hashCode() : 0); result = 31 * result + (filePath != null ? filePath.hashCode() : 0); return result; } } protected PathTokens checkUri(URI uri) { Matcher matcher = URI_PATTERN.matcher(uri.toString()); if (!matcher.matches()) { throw new IllegalArgumentException("URI does not match this provider: " + uri); } String contextId = matcher.group(2); String path = matcher.group(3); PathTokens tokens = new PathTokens(); tokens.name = contextId; tokens.contextId = contextId; tokens.filePath = path; return tokens; } protected PathTokens resolveUri(URI uri, String defContextId) { PathTokens tokens = checkUri(uri); if (tokens.contextId == null) { tokens.contextId = defContextId; return tokens; } Matcher matcher = CONTEXT_ID_PATTERN.matcher(tokens.contextId); if (!matcher.matches()) { // look for this container name by invoking the remote API try { List<Map<String, Object>> found = api.projectFind(tokens.contextId); if (found == null || found.size() != 1) { throw new IllegalStateException( String.format("Unable to retrieve project-id by name: '%s' (1)", tokens.contextId)); } tokens.contextId = found.get(0).get("id").toString(); } catch (IOException e) { throw new IllegalStateException( String.format("Unable to retrieve project-id by name: '%s' (2)", tokens.contextId), e); } } return tokens; } @Override public final FileSystem getFileSystem(URI uri) { log.trace("Parsing URI: {}", uri); PathTokens tokens = resolveUri(uri, defaultContextId); return fileSystems.get(tokens.contextId); } @Override public DirectoryStream<Path> newDirectoryStream(Path dir, Filter<? super Path> filter) throws IOException { return new DxDirectoryStream(toDxPath(dir), filter); } public InputStream newInputStream(Path file, OpenOption... options) throws IOException { if (options.length > 0) { for (OpenOption opt : options) { if (opt != StandardOpenOption.READ) throw new UnsupportedOperationException("'" + opt + "' not allowed"); } } final DxPath path = toDxPath(file); final String fileId = path.getFileId(); final Map<String, Object> download = api.fileDownload(fileId); final String url = (String) download.get("url"); final Map<String, String> headers = (Map<String, String>) download.get("headers"); final HttpClient client = DxHttpClient.getInstance().http(); client.getParams().setParameter(CoreProtocolPNames.PROTOCOL_VERSION, HttpVersion.HTTP_1_1); HttpGet get = new HttpGet(url); get.getParams().setParameter(ClientPNames.COOKIE_POLICY, CookiePolicy.IGNORE_COOKIES); for (Map.Entry<String, String> item : headers.entrySet()) { get.setHeader(item.getKey(), item.getValue()); } return client.execute(get).getEntity().getContent(); } private static void checkAllowedOptions(Set<? extends OpenOption> allowed, OpenOption... options) { if (options == null) return; for (OpenOption opt : options) { if (!allowed.contains(opt)) { throw new UnsupportedOperationException(opt.toString() + " options not allowed"); } } } private static Set<? extends OpenOption> OUTPUT_STREAM_VALID_OPTIONS = new HashSet<>( Arrays.asList(StandardOpenOption.CREATE, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)); /** * * @param path * @param options * @return * @throws IOException */ public OutputStream newOutputStream(Path path, OpenOption... options) throws IOException { checkAllowedOptions(OUTPUT_STREAM_VALID_OPTIONS, options); // create the file final DxPath thePath = toDxPath(path); final DxFileSystem theFileSystem = thePath.getFileSystem(); final String fileId = theFileSystem.fileNew(thePath); // set the type accordingly thePath.fileId = fileId; thePath.type = DxPath.PathType.FILE; // create the output stream uploaded return new DxUploadOutputStream(fileId, api); } /** * Operation not supported * * @throws UnsupportedOperationException */ @Override public FileChannel newFileChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException { throw new UnsupportedOperationException(); } /** * Operation not supported * * @throws UnsupportedOperationException */ @Override public SeekableByteChannel newByteChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException { throw new UnsupportedOperationException(); } /** * Create a new *remote* folder * <p> * * See https://wiki.dnanexus.com/API-Specification-v1.0.0/Folders%20and%20Deletion#API-method:-/class-xxxx/newFolder * * @param path * @param attrs * @throws IOException */ @Override public void createDirectory(Path path, FileAttribute<?>... attrs) throws IOException { if (attrs.length > 0) { throw new UnsupportedOperationException( "Attributes on directories are not supported by DnaNexus file system"); } DxPath dxPath = toDxPath(path); DxFileSystem theFileSystem = dxPath.getFileSystem(); theFileSystem.createFolder(dxPath); dxPath.type = DxPath.PathType.DIRECTORY; } /** * Deletes a file. This method works in exactly the manner specified by the * {@link Files#delete} method. * * @param path * the path to the file to delete * * @throws java.nio.file.NoSuchFileException * if the file does not exist <i>(optional specific exception)</i> * @throws java.nio.file.DirectoryNotEmptyException * if the file is a directory and could not otherwise be deleted * because the directory is not empty <i>(optional specific * exception)</i> * <p> * See http://wiki.dnanexus.com/API-Specification-v1.0.0/Folders-and-Deletion#API-method:-/class-xxxx/removeObjects * * @param path * @throws IOException */ @Override public void delete(Path path) throws IOException { DxPath dxPath = toDxPath(path); DxFileSystem dxFileSystem = dxPath.getFileSystem(); DxFileAttributes attr = dxPath.readAttributes(); if (attr.isDirectory()) { dxFileSystem.deleteFolder(dxPath, false); } else { dxFileSystem.deleteFiles(dxPath); } // clear all the attributes on this file since does not exist any more dxPath.clearAttributes(); } public void deleteDir(DxPath path, boolean recurse) throws IOException { path.getFileSystem().deleteFolder(path, recurse); } /** * Implements the *copy* operation using the DnaNexus API *clone* * * * <p> * See clone https://wiki.dnanexus.com/API-Specification-v1.0.0/Cloning#API-method%3A-%2Fclass-xxxx%2Fclone * * @param source * @param target * @param options * @throws IOException */ @Override public void copy(Path source, Path target, CopyOption... options) throws IOException { List<CopyOption> opts = Arrays.asList(options); boolean targetExists = Files.exists(target); if (targetExists) { if (Files.isRegularFile(target)) { if (opts.contains(StandardCopyOption.REPLACE_EXISTING)) { Files.delete(target); } else { throw new FileAlreadyExistsException("Copy failed -- target file already exists: " + target); } } else if (Files.isDirectory(target)) { target = target.resolve(source.getFileName()); } else { throw new UnsupportedOperationException(); } } String name1 = source.getFileName().toString(); String name2 = target.getFileName().toString(); if (!name1.equals(name2)) { throw new UnsupportedOperationException( "Copy to a file with a different name is not supported: " + source.toString()); } final DxPath dxSource = toDxPath(source); final DxFileSystem dxFileSystem = dxSource.getFileSystem(); dxFileSystem.fileCopy(dxSource, toDxPath(target)); } // TODO move @Override public void move(Path source, Path target, CopyOption... options) throws IOException { // see: // container move https://wiki.dnanexus.com/API-Specification-v1.0.0/Folders-and-Deletion#API-method%3A-%2Fclass-xxxx%2Fmove // container rename https://wiki.dnanexus.com/API-Specification-v1.0.0/Folders-and-Deletion#API-method%3A-%2Fclass-xxxx%2FrenameFolder // file rename https://wiki.dnanexus.com/API-Specification-v1.0.0/Name#API-method%3A-%2Fclass-xxxx%2Frename } @Override public boolean isSameFile(Path path, Path path2) throws IOException { return path.normalize().compareTo(path2.normalize()) == 0; } @Override public boolean isHidden(Path path) throws IOException { return readAttributes(path, DxFileAttributes.class).isHidden(); } //TODO getFileStore @Override public FileStore getFileStore(Path path) throws IOException { throw new UnsupportedOperationException(); } /** * * TODO checkAccess * http://openjdk.java.net/projects/nio/javadoc/java/nio/file/spi/FileSystemProvider.html#checkAccess(java.nio.file.Path, java.nio.file.AccessMode...) * * @param path * @param modes * @throws IOException */ @Override public void checkAccess(Path path, AccessMode... modes) throws IOException { toDxPath(path).readAttributes(); } @Override @SuppressWarnings("unchecked") public <V extends FileAttributeView> V getFileAttributeView(Path path, Class<V> type, LinkOption... options) { if (type == DxFileAttributeView.class) { return (V) new DxFileAttributeView(toDxPath(path)); } return null; } @Override @SuppressWarnings("unchecked") public <V extends BasicFileAttributes> V readAttributes(Path path, Class<V> type, LinkOption... options) throws IOException { if (type == BasicFileAttributes.class || type == DxFileAttributes.class) { DxFileAttributeView view = new DxFileAttributeView(toDxPath(path)); return (V) view.readAttributes(); } return null; } @Override public Map<String, Object> readAttributes(Path path, String attributes, LinkOption... options) throws IOException { int pos = attributes.indexOf(':'); if (pos != -1) { String view = attributes.substring(0, pos++); if (!view.equals(DxFileAttributeView.NAME)) { throw new IllegalArgumentException( String.format("Illegal view for DnaNexus file system: '%s'", view)); } attributes = attributes.substring(pos); } DxFileAttributeView view = new DxFileAttributeView(toDxPath(path)); return view.readAttributes(attributes); } @Override public void setAttribute(Path path, String attribute, Object value, LinkOption... options) throws IOException { final DxPath dxPath = toDxPath(path); final DxFileSystem dxFileSystem = dxPath.getFileSystem(); if (attribute.equals("tags")) { dxFileSystem.fileAddTags(dxPath, (String[]) value); } else if (attribute.equals("types")) { dxFileSystem.fileAddTypes(dxPath, (String[]) value); } else { throw new UnsupportedOperationException(String.format("Attribute '%s' cannot be changed", attribute)); } } // Checks that the given file is a UnixPath static final DxPath toDxPath(Path path) { if (path == null) { throw new NullPointerException(); } if (!(path instanceof DxPath)) { throw new ProviderMismatchException(); } return (DxPath) path; } /** * @return The current installed instance of the {@code DxFileSystemProvider} or {@code null} if * no DX provider is installed */ static DxFileSystemProvider instance = null; static DxFileSystemProvider defaultInstance() { if (instance != null) return instance; for (FileSystemProvider provider : FileSystemProvider.installedProviders()) { if (provider instanceof DxFileSystemProvider) { return instance = (DxFileSystemProvider) provider; } } return null; } }