Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.openflamingo.util; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.openflamingo.core.exception.FileSystemException; import org.slf4j.helpers.MessageFormatter; import org.springframework.util.FileCopyUtils; import java.io.File; import java.io.IOException; import java.net.InetAddress; import java.net.URI; import java.net.UnknownHostException; /** * File System Utility. * * @author Edward KIM * @since 0.2 */ public class FileSystemUtils { /** * . <tt>hdfs://</tt> ? <tt>file:</tt> * ? <tt>file:</tt>? ?. * * @param path ? ? ? * @return ? */ public static String correctPath(String path) { if (path == null || path.length() < 1) { return null; } Path filePath = new Path(path); String scheme = filePath.toUri().getScheme(); if (scheme == null) { return "file:" + path; } return path; } /** * . <tt>hdfs://</tt> ? <tt>file:</tt> * ? <tt>file:</tt>? ?. * * @param parent * @param child ?? * @return ? */ public static String correctPath(String parent, String child) { if (StringUtils.isEmpty(parent) || StringUtils.isEmpty(child)) { return null; } String path = parent + "/" + child; return correctPath(path); } /** * . <tt>hdfs://</tt> ? <tt>file:</tt> * ? <tt>file:</tt>? ?. * * @param parent * @param prefix ?? prefix * @param filename ? * @return ? */ public static String correctPath(String parent, String prefix, String filename) { if (StringUtils.isEmpty(parent) || StringUtils.isEmpty(prefix) || StringUtils.isEmpty(filename)) { return null; } String path = parent + "/" + prefix + "_" + filename; return correctPath(path); } /** * ? ? ? ?. * * @param path * @param scheme ? ? scheme(; hdfs:// file:) */ public static void checkScheme(String path, FileSystemScheme scheme) { if (path == null || path.length() < 1) { return; } if (!path.startsWith(scheme.getScheme())) { throw new FileSystemException( ExceptionUtils.getMessage("Invalid file system '{}' => Path '{}'", scheme.getScheme(), path)); } } /** * ? ? ?. * * @param path */ public static void testCorrentAndCreateDir(String path) { String correctedPath = correctPath(path); testCreateDir(new Path(correctedPath)); } /** * ?. * ??? ?. * * @param path * @throws org.openflamingo.core.exception.FileSystemException ? , ? ?? */ public static void testCreateDir(Path path) { try { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); // FileSystem fs = path.getFileSystem(conf); System.out.println("--------New------------------"); System.out.println("--------fs--------" + fs); System.out.println("--------fs getUri--------" + fs.getUri()); System.out.println("--------fs getWorkingDirectory--------" + fs.getWorkingDirectory()); System.out.println("--------fs getHomeDirectory--------" + fs.getHomeDirectory()); System.out.println("--------path--------" + path); System.out.println("--------fs.exists(path)--------" + fs.exists(path)); System.out.println("--------fs.mkdirs(path--------" + fs.mkdirs(path)); if (fs.exists(path) && !fs.getFileStatus(path).isDir()) { throw new FileSystemException(ExceptionUtils.getMessage("'{}' is not directory.", path)); } if (!fs.exists(path)) { if (!fs.mkdirs(path)) { throw new FileSystemException(ExceptionUtils.getMessage("Cannot create '{}'", path)); } } } catch (Exception ex) { String message = MessageFormatter.format("Cannot create '{}'", path.toString()).getMessage(); throw new FileSystemException(message, ex); } } /** * ? ? ? . * * @param path ? ? ? * @return ? */ public static FileSystem getFileSystem(Path path) { try { Configuration conf = new Configuration(); return path.getFileSystem(conf); } catch (Exception ex) { throw new FileSystemException(ExceptionUtils.getMessage("Cannot get file system of '{}'", path), ex); } } /** * ? ? ? . * * @param path ? ? ? * @return ? */ public static FileSystem getFileSystem(String path) { return getFileSystem(new Path(path)); } /** * ? ? ? ? ?? ?. * * @param path1 1 * @param path2 2 */ public static void validateSameFileSystem(String path1, String path2) { Path p1 = new Path(correctPath(path1)); Path p2 = new Path(correctPath(path2)); FileSystem fs1 = null; FileSystem fs2 = null; try { fs1 = p1.getFileSystem(new Configuration()); fs2 = p2.getFileSystem(new Configuration()); } catch (Exception ex) { throw new FileSystemException(ExceptionUtils.getMessage("Cannot access '{}' or '{}'.", p1, p2), ex); } if (!compareFs(fs1, fs2)) { throw new FileSystemException(ExceptionUtils.getMessage("File system is not same : {}, {}", p1, p2)); } if (p1.equals(p2)) { throw new FileSystemException(ExceptionUtils.getMessage("Same path : {}, {}", p1, p2)); } } /** * ? ? ? URI? scheme? ? ?? ?. * * @param fs1 ?1 * @param fs2 ?2 * @return ?? <tt>true</tt> */ private static boolean compareFs(FileSystem fs1, FileSystem fs2) { URI uri1 = fs1.getUri(); URI uri2 = fs2.getUri(); if (uri1.getScheme() == null) { return false; } if (!uri1.getScheme().equals(uri2.getScheme())) { return false; } String srcHost = uri1.getHost(); String dstHost = uri2.getHost(); if ((srcHost != null) && (dstHost != null)) { try { srcHost = InetAddress.getByName(srcHost).getCanonicalHostName(); dstHost = InetAddress.getByName(dstHost).getCanonicalHostName(); } catch (UnknownHostException ue) { return false; } if (!srcHost.equals(dstHost)) { return false; } } else if (srcHost == null && dstHost != null) { return false; } else if (srcHost != null) { return false; } // ? ? return uri1.getPort() == uri2.getPort(); } /** * ? ?? . ? ?? ? ?? . * ? ? ? ? ?? . * * @param sourceFilePath ?? * @return ?? * @throws org.openflamingo.core.exception.FileSystemException ?? */ public static String load(String sourceFilePath) { String workingDirectory = System.getProperty("user.dir"); try { if (new File(sourceFilePath).exists()) { String filePath = correctPath(sourceFilePath); return loadFromFile(filePath); } else { String filePath = new File(workingDirectory, sourceFilePath).getAbsolutePath(); return loadFromFile(filePath); } } catch (IOException ex) { throw new FileSystemException(ExceptionUtils.getMessage("Cannot load '{}'.", sourceFilePath), ex); } } /** * ? ?? . ? ?? ? ?? . * ? ? ? ? ?? . * * @param sourceFilePath ?? * @return ?? * @throws org.openflamingo.core.exception.FileSystemException ?? */ public static byte[] loadBytes(String sourceFilePath) { String workingDirectory = System.getProperty("user.dir"); try { if (new File(sourceFilePath).exists()) { String filePath = correctPath(sourceFilePath); return loadBytesFromFile(filePath); } else { String filePath = new File(workingDirectory, sourceFilePath).getAbsolutePath(); return loadBytesFromFile(filePath); } } catch (IOException ex) { throw new FileSystemException(ExceptionUtils.getMessage("Cannot load '{}'.", sourceFilePath), ex); } } /** * ? ?? . ? ?? ? ?? . * ? ? ? ? ?? . * * @param sourceFilePath ?? * @return ?? * @throws org.openflamingo.core.exception.FileSystemException ?? */ public static String loadFromFile(String sourceFilePath) throws IOException { byte[] bytes = loadBytesFromFile(sourceFilePath); return new String(bytes); } /** * ? ?? . ? ?? ? ?? . * ? ? ? ? ?? . * * @param sourceFilePath ?? * @return ?? * @throws org.openflamingo.core.exception.FileSystemException ?? */ public static byte[] loadBytesFromFile(String sourceFilePath) throws IOException { FileSystem fs = getFileSystem(sourceFilePath); FSDataInputStream is = fs.open(new Path(sourceFilePath)); byte[] bytes = FileCopyUtils.copyToByteArray(is); IOUtils.closeQuietly(is); return bytes; } /** * ? ?? . ? ?? ? ?? . * ? ? ? ? ?? . * * @param sourceFileSystem ? * @param sourceFilePath ?? * @return ?? * @throws org.openflamingo.core.exception.FileSystemException ?? */ public static byte[] loadBytesFromFile(FileSystem sourceFileSystem, String sourceFilePath) throws IOException { FSDataInputStream is = sourceFileSystem.open(new Path(sourceFilePath)); byte[] bytes = FileCopyUtils.copyToByteArray(is); IOUtils.closeQuietly(is); return bytes; } /** * ? ?? . ? ?? * ?? . * <tt>path</tt> ? ? ?? ? ?? ? ?. * * @param source ?? * @param targetFilePath ?? * @return ? (<tt>true</tt> ?? ) * @throws org.openflamingo.core.exception.FileSystemException ?? */ public static boolean save(byte[] source, String targetFilePath) { String workingDirectory = System.getProperty("user.dir"); try { // ? ?? ? ?? . if (!new File(targetFilePath).exists()) { String filePath = new File(workingDirectory, targetFilePath).getAbsolutePath(); return saveToFile(source, filePath); } else { String filePath = new File(targetFilePath).getAbsolutePath(); return saveToFile(source, filePath); } } catch (Exception ex) { throw new FileSystemException(ExceptionUtils.getMessage("Cannot save '{}'.", targetFilePath), ex); } } /** * ? ?? ?. * * @param source ?? * @param targetFilePath ?? * @return ? (<tt>true</tt> ?? ) */ public static boolean saveToFile(byte[] source, String targetFilePath) { try { FileSystem fs = getFileSystem(targetFilePath); FSDataOutputStream fos = fs.create(new Path(targetFilePath)); FileCopyUtils.copy(source, fos); IOUtils.closeQuietly(fos); return fs.exists(new Path(targetFilePath)); } catch (Exception ex) { throw new FileSystemException(ExceptionUtils.getMessage("Cannot save '{}'.", targetFilePath), ex); } } /** * ? ?? ?. * * @param targetFileSystem ?? ? ? * @param source ?? * @param targetFilePath ?? * @return ? (<tt>true</tt> ?? ) * @throws java.io.IOException ?? */ public static boolean saveToFile(byte[] source, FileSystem targetFileSystem, String targetFilePath) throws IOException { FSDataOutputStream fos = targetFileSystem.create(new Path(targetFilePath)); FileCopyUtils.copy(source, fos); IOUtils.closeQuietly(fos); return targetFileSystem.exists(new Path(targetFilePath)); } }