Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one or more contributor license * agreements. See the NOTICE file distributed with this work for additional information regarding * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. You may obtain a * copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.aliyun.odps.volume; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import com.aliyun.odps.Odps; import com.aliyun.odps.OdpsException; import com.aliyun.odps.Volume; import com.aliyun.odps.VolumeException; import com.aliyun.odps.VolumeFSFile; import com.aliyun.odps.Volumes; import com.aliyun.odps.fs.VolumeFileSystemConfigKeys; import com.aliyun.odps.tunnel.TunnelException; import com.aliyun.odps.tunnel.VolumeFSErrorCode; import com.aliyun.odps.tunnel.VolumeFSTunnel; import com.aliyun.odps.tunnel.io.CompressOption; import com.aliyun.odps.tunnel.io.VolumeOutputStream; import com.aliyun.odps.volume.protocol.VolumeFSConstants; import com.aliyun.odps.volume.protocol.VolumeFSErrorMessageGenerator; /** * Client for Volume * * @author Emerson Zhao [mailto:zhenyi.zzy@alibaba-inc.com] * */ public class VolumeFSClient { private Odps odps; private String tunnelEndpoint; private Configuration conf; public VolumeFSClient(Odps odps, String project, String serviceEndpoint, String tunnelEndpoint, Configuration conf) { this.odps = odps; this.odps.setDefaultProject(project); this.odps.setEndpoint(serviceEndpoint); this.tunnelEndpoint = tunnelEndpoint; this.conf = conf; } /** * Get File meta info * * @param path * @throws VolumeException */ public VolumeFSFile getFileInfo(String path) throws VolumeException { return new VolumeFSJobRunnerProxy<VolumeFSFile>() { @Override public VolumeFSFile doJob(String path, Map<String, Object> params) throws VolumeException { VolumeFSFile file = odps.volumes().get(VolumeFSUtil.getVolumeFromPath(path)).getVolumeFSFile(path); try { file.reload(); } catch (OdpsException e) { throw new VolumeException(e); } return file; } @Override public VolumeFSFile onVolumeMissing(String path, Map<String, Object> params) throws VolumeException { VolumeFSFile file = VolumeFSFile.getRoot(odps.getDefaultProject(), odps); return file; } @Override public VolumeFSFile onNoSuchVolume(String path, Map<String, Object> params) throws VolumeException { throw new VolumeException(VolumeFSErrorCode.NoSuchPath, VolumeFSErrorMessageGenerator.noSuchFileOrDirectory(path)); } @Override public VolumeFSFile onInvalidPath(String path, Map<String, Object> params) throws VolumeException { if (VolumeFSUtil.checkPathIsJustVolume(path)) { try { if (!odps.volumes().exists(VolumeFSUtil.getVolumeFromPath(path))) { throw new VolumeException(VolumeFSErrorCode.NoSuchPath, VolumeFSErrorMessageGenerator.noSuchFileOrDirectory(path)); } } catch (OdpsException e) { throw new VolumeException(e); } Volume v = odps.volumes().get(VolumeFSUtil.getVolumeFromPath(path)); if (isNewVolume(v)) { return VolumeFSFile.transferVolumeToVolumeFSFile(odps.getDefaultProject(), v, odps.getRestClient()); } else { throw new VolumeException(VolumeFSErrorCode.NoSuchPath, VolumeFSErrorMessageGenerator.oldVolumeAlert(VolumeFSUtil.getVolumeFromPath(path))); } } else { throw (VolumeException) params.get(CUR_EXCEPTION); } } }.run(path, null); } /** * Make directory * * @param path * @throws VolumeException */ public boolean mkdirs(String path) throws VolumeException { return new VolumeFSJobRunnerProxy<Boolean>() { @Override public Boolean doJob(String path, Map<String, Object> params) throws VolumeException { VolumeFSFile.create(odps.getDefaultProject(), path, true, odps.getRestClient()); return true; } @Override public Boolean onVolumeMissing(String path, Map<String, Object> params) throws VolumeException { throw new VolumeException(VolumeFSErrorCode.PathAlreadyExists, VolumeFSErrorMessageGenerator.fileExists(path)); } @Override public Boolean onNoSuchVolume(String path, Map<String, Object> params) throws VolumeException { try { odps.volumes().create(VolumeFSUtil.getVolumeFromPath(path), VolumeFSConstants.CREATE_VOLUME_COMMENT, Volume.Type.NEW); } catch (Exception e) { throw new VolumeException(e); } if (VolumeFSUtil.checkPathIsJustVolume(path)) { return true; } else { return doJob(path, params); } } @Override public Boolean onInvalidPath(String path, Map<String, Object> params) throws VolumeException { // If the path is a volume name , will return true if (VolumeFSUtil.checkPathIsJustVolume(path)) { try { if (!odps.volumes().exists(VolumeFSUtil.getVolumeFromPath(path))) { odps.volumes().create(VolumeFSUtil.getVolumeFromPath(path), VolumeFSConstants.CREATE_VOLUME_COMMENT, Volume.Type.NEW); } } catch (OdpsException e) { throw new VolumeException(e); } return true; } else { throw (VolumeException) params.get(CUR_EXCEPTION); } } @Override public Boolean onPathAlreadyExists(String path, Map<String, Object> params) throws VolumeException { VolumeFSFile file = odps.volumes().get(VolumeFSUtil.getVolumeFromPath(path)).getVolumeFSFile(path); try { file.reload(); } catch (OdpsException oe) { throw new VolumeException(oe); } if (Boolean.TRUE.equals(file.getIsdir())) { return Boolean.TRUE; } else { throw (VolumeException) params.get(CUR_EXCEPTION); } } }.run(path, null); } /** * Set file's replication * * @param path * @param replication * @throws VolumeException */ public boolean setReplication(String path, short replication) throws VolumeException { Map<String, Object> params = new HashMap<String, Object>(); Map<String, String> innerParams = new HashMap<String, String>(); innerParams.put(VolumeFSFile.ParamKey.REPLICATION.name().toLowerCase(), String.valueOf(replication)); params.put("params", innerParams); return new VolumeFSJobRunnerProxy<Boolean>() { @SuppressWarnings("unchecked") @Override public Boolean doJob(String path, Map<String, Object> params) throws VolumeException { VolumeFSFile file = odps.volumes().get(VolumeFSUtil.getVolumeFromPath(path)).getVolumeFSFile(path); file.update((Map<String, String>) params.get("params")); return true; } @Override public Boolean onVolumeMissing(String path, Map<String, Object> params) throws VolumeException { throw new VolumeException(VolumeFSErrorCode.NotAcceptableOperation, VolumeFSErrorMessageGenerator.isADirectory(path)); } @Override public Boolean onNoSuchVolume(String path, Map<String, Object> params) throws VolumeException { throw new VolumeException(VolumeFSErrorCode.NoSuchPath, VolumeFSErrorMessageGenerator.noSuchFileOrDirectory(path)); } }.run(path, params); } /** * List files (and directories) in specific path * * @param path * @throws VolumeException */ public VolumeFSFile[] getFileInfosByPath(String path) throws VolumeException { return new VolumeFSJobRunnerProxy<VolumeFSFile[]>() { @Override public VolumeFSFile[] doJob(String path, Map<String, Object> params) throws VolumeException { VolumeFSFile file = odps.volumes().get(VolumeFSUtil.getVolumeFromPath(path)).getVolumeFSFile(path); Iterator<VolumeFSFile> iterator = file.iterator(); List<VolumeFSFile> files = new ArrayList<VolumeFSFile>(); try { while (iterator.hasNext()) { files.add(iterator.next()); } } catch (RuntimeException e) { if (e.getCause() instanceof VolumeException) { throw (VolumeException) e.getCause(); } else { throw new VolumeException(e); } } return files.toArray(new VolumeFSFile[0]); } @Override public VolumeFSFile[] onVolumeMissing(String path, Map<String, Object> params) throws VolumeException { List<VolumeFSFile> files = new ArrayList<VolumeFSFile>(); Volumes volumes = odps.volumes(); Iterator<Volume> it = volumes.iterator(); while (it.hasNext()) { Volume v = it.next(); if (isNewVolume(v)) { files.add(VolumeFSFile.transferVolumeToVolumeFSFile(odps.getDefaultProject(), v, odps.getRestClient())); } } return files.toArray(new VolumeFSFile[0]); } @Override public VolumeFSFile[] onNoSuchVolume(String path, Map<String, Object> params) throws VolumeException { throw new VolumeException(VolumeFSErrorCode.NoSuchPath, VolumeFSErrorMessageGenerator.noSuchFileOrDirectory(path)); } @Override public VolumeFSFile[] onInvalidPath(String path, Map<String, Object> params) throws VolumeException { if (VolumeFSUtil.checkPathIsJustVolume(path)) { return doJob(path, params); } else { throw (VolumeException) params.get(CUR_EXCEPTION); } } }.run(path, null); } /** * Rename file (or directory) * * @param src * @param dst * @throws VolumeException */ public boolean rename(String src, String dst) throws VolumeException { Map<String, Object> params = new HashMap<String, Object>(); Map<String, String> innerParams = new HashMap<String, String>(); innerParams.put(VolumeFSFile.ParamKey.PATH.name().toLowerCase(), dst); params.put("params", innerParams); return new VolumeFSJobRunnerProxy<Boolean>() { @SuppressWarnings("unchecked") @Override public Boolean doJob(String path, Map<String, Object> params) throws VolumeException { VolumeFSFile file = odps.volumes().get(VolumeFSUtil.getVolumeFromPath(path)).getVolumeFSFile(path); file.update((Map<String, String>) params.get("params")); return true; } @Override public Boolean onVolumeMissing(String path, Map<String, Object> params) throws VolumeException { return false; } @Override public Boolean onNoSuchVolume(String path, Map<String, Object> params) throws VolumeException { throw new VolumeException(VolumeFSErrorCode.NoSuchPath, VolumeFSErrorMessageGenerator.noSuchFileOrDirectory(path)); } @Override public Boolean onInvalidPath(String path, Map<String, Object> params) throws VolumeException { if (VolumeFSUtil.checkPathIsJustVolume(path)) { throw new VolumeException(VolumeFSErrorCode.NotAcceptableOperation, VolumeFSErrorMessageGenerator .theOpreationIsNotAllowed("mv volume (The first level directory)")); } else { throw (VolumeException) params.get(CUR_EXCEPTION); } } }.run(src, params); } /** * Delete file (or directory) * * @param path * @param recursive * @throws VolumeException */ public boolean delete(String path, boolean recursive) throws VolumeException { Map<String, Object> params = new HashMap<String, Object>(); params.put(VolumeFSFile.ParamKey.RECURSIVE.name().toLowerCase(), recursive); return new VolumeFSJobRunnerProxy<Boolean>() { @Override public Boolean doJob(String path, Map<String, Object> params) throws VolumeException { VolumeFSFile file = odps.volumes().get(VolumeFSUtil.getVolumeFromPath(path)).getVolumeFSFile(path); file.delete((Boolean) params.get(VolumeFSFile.ParamKey.RECURSIVE.name().toLowerCase())); return true; } @Override public Boolean onVolumeMissing(String path, Map<String, Object> params) throws VolumeException { return false; } @Override public Boolean onNoSuchVolume(String path, Map<String, Object> params) throws VolumeException { throw new VolumeException(VolumeFSErrorCode.NoSuchPath, VolumeFSErrorMessageGenerator.noSuchFileOrDirectory(path)); } @Override public Boolean onInvalidPath(String path, Map<String, Object> params) throws VolumeException { // If the path is a volume name , will call volume API if (VolumeFSUtil.checkPathIsJustVolume(path)) { try { Volume v = odps.volumes().get(VolumeFSUtil.getVolumeFromPath(path)); if (isNewVolume(v)) { odps.volumes().delete(VolumeFSUtil.getVolumeFromPath(path)); return true; } else { throw new VolumeException(VolumeFSErrorCode.NoSuchPath, VolumeFSErrorMessageGenerator .oldVolumeAlert(VolumeFSUtil.getVolumeFromPath(path))); } } catch (OdpsException e) { throw new VolumeException(e); } } else { throw (VolumeException) params.get(CUR_EXCEPTION); } } }.run(path, params); } /** * Download volume file * * @param path * @param start start of range * @param end start of range * @param targetFile * @param append * @throws IOException */ public void downloadFile(String path, Long start, Long end, File targetFile, boolean append) throws VolumeException { VolumeFSUtil.checkPath(path); OutputStream outputStream = null; try { outputStream = new FileOutputStream(targetFile, append); } catch (FileNotFoundException e) { throw new VolumeException(e); } InputStream inputStream = null; try { inputStream = openInputStream(path, start, end); IOUtils.copy(inputStream, outputStream); } catch (IOException e) { throw new VolumeException(e); } finally { IOUtils.closeQuietly(outputStream); IOUtils.closeQuietly(inputStream); } } /** * Create an {@link InputStream} at the indicated Path * * @param path * @param start * @param end * @throws IOException */ public InputStream openInputStream(String path, Long start, Long end) throws VolumeException { VolumeFSUtil.checkPath(path); boolean compress = conf.getBoolean(VolumeFileSystemConfigKeys.ODPS_VOLUME_TRANSFER_COMPRESS_ENABLED, false); CompressOption compressOption = null; if (compress) { CompressOption.CompressAlgorithm a = CompressOption.CompressAlgorithm .valueOf(conf.get(VolumeFileSystemConfigKeys.ODPS_VOLUME_TRANSFER_COMPRESS_ALGORITHM, VolumeFSConstants.ODPS_ZLIB)); compressOption = new CompressOption(a, 1, 0); } InputStream in = null; try { in = getVolumeTunnel().openInputStream(odps.getDefaultProject(), path, start, end, compressOption); } catch (TunnelException e) { throw new VolumeException(e); } return in; } /** * Create an {@link OutputStream} at the indicated Path * * @param path * @param overwrite * @param replication * @throws IOException */ public OutputStream openOutputStream(String path, boolean overwrite, short replication) throws VolumeException { VolumeFSUtil.checkPath(path); VolumeFSFile file = null; try { if (!odps.volumes().exists(VolumeFSUtil.getVolumeFromPath(path))) { odps.volumes().create(VolumeFSUtil.getVolumeFromPath(path), VolumeFSConstants.CREATE_VOLUME_COMMENT, Volume.Type.NEW); } } catch (OdpsException oe) { throw new VolumeException(oe); } try { file = getFileInfo(path); } catch (VolumeException e) { if (!VolumeFSErrorCode.NoSuchPath.equalsIgnoreCase(e.getErrCode())) { throw e; } } if (file != null) { if (overwrite) { if (file.getIsdir()) { throw new VolumeException(VolumeFSErrorCode.PathAlreadyExists, VolumeFSErrorMessageGenerator.fileExists(path)); } else { delete(path, true); } } else { throw new VolumeException(VolumeFSErrorCode.PathAlreadyExists, VolumeFSErrorMessageGenerator.fileExists(path)); } } VolumeFSTunnel tunnel = getVolumeTunnel(); // create a zero-byte file to satisfy visibility before wrote data into file createZeroByteFile(path, replication, tunnel); boolean compress = conf.getBoolean(VolumeFileSystemConfigKeys.ODPS_VOLUME_TRANSFER_COMPRESS_ENABLED, false); CompressOption compressOption = null; if (compress) { CompressOption.CompressAlgorithm a = CompressOption.CompressAlgorithm .valueOf(conf.get(VolumeFileSystemConfigKeys.ODPS_VOLUME_TRANSFER_COMPRESS_ALGORITHM, VolumeFSConstants.ODPS_ZLIB)); compressOption = new CompressOption(a, 1, 0); } OutputStream out = null; try { out = tunnel.openOutputStream(odps.getDefaultProject(), path, new Integer(replication), compressOption); } catch (TunnelException e) { throw new VolumeException(e); } return out; } /** * Create a zero-byte file * * @param path * @param replication * @param tunnel * @throws VolumeException */ public void createZeroByteFile(String path, short replication, VolumeFSTunnel tunnel) throws VolumeException { VolumeFSUtil.checkPath(path); OutputStream o = null; try { o = tunnel.openOutputStream(odps.getDefaultProject(), path, new Integer(replication), new CompressOption()); String sessionId = VolumeFSTunnel.getUploadSessionId((VolumeOutputStream) o); tunnel.commit(odps.getDefaultProject(), path, sessionId); } catch (TunnelException e) { throw new VolumeException(e); } } /** * Commit the uploadSession * * @param path * @param out * @param overwrite * @throws VolumeException */ public void commitUploadSession(String path, VolumeOutputStream out, boolean overwrite) throws VolumeException { VolumeFSUtil.checkPath(path); try { out.close(); String sessionId = VolumeFSTunnel.getUploadSessionId(out); VolumeFSFile file = null; try { file = getFileInfo(path); } catch (Exception e) { } if (file != null) { if (overwrite) { if (file.getIsdir()) { throw new VolumeException(VolumeFSErrorCode.PathAlreadyExists, VolumeFSErrorMessageGenerator.fileExists(path)); } else { file.delete(true); } } else { if (file.getLength().longValue() == 0) { // Delete zero-byte file file.delete(true); } else { throw new VolumeException(VolumeFSErrorCode.PathAlreadyExists, VolumeFSErrorMessageGenerator.fileExists(path)); } } } getVolumeTunnel().commit(odps.getDefaultProject(), path, sessionId); } catch (Exception e) { throw new VolumeException(e); } } private VolumeFSTunnel getVolumeTunnel() { VolumeFSTunnel tunnelFS = new VolumeFSTunnel(odps.clone()); if (tunnelEndpoint != null) tunnelFS.setEndpoint(tunnelEndpoint); return tunnelFS; } private boolean isNewVolume(Volume v) { return Volume.Type.NEW.equals(v.getType()); } }