com.jeffy.hdfs.HDFSReadFile.java Source code

Introduction

Here is the source code for com.jeffy.hdfs.HDFSReadFile.java

Source

/*
 * Copyright AsiaInfo Authors.
 * 
 * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
 */
package com.jeffy.hdfs;

import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsUrlStreamHandlerFactory;
import org.apache.hadoop.fs.Path;

/**
 * HDFS??
 * Hadoop 2.7?
 * 
 * @author Jeffy<renwu58@gmail.com>
 */
public class HDFSReadFile {

    /**
     * ?URLHandler Factoryhdfs??
     * java.net.MalformedURLException: unknown protocol: hdfs
     */
    static {
        URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
    }

    /**
     * @param args
     */
    public static void main(String[] args) {
        String path = "hdfs://master:8020/tmp/user.csv";
        //HDFSReadFile.readDataUseURL(path);
        try {
            HDFSReadFile.readDataUseFileSystem(path);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * java.net.URL?HDFS??
     * 
     * @param path  hdfs://master:8020/tmp/user.csv
     */
    public static void readDataUseURL(String path) {
        try (InputStream in = new URL(path).openStream()) {
            IOUtils.copy(in, System.out);
        } catch (MalformedURLException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * HadoopFileSystem API??
     * 
     * @param path
     * @throws IOException 
     */
    public static void readDataUseFileSystem(String path) throws IOException {

        Configuration config = new Configuration();
        /**
         * ?FileSystem????????
         *  public static FileSystem get(Configuration conf) throws IOException
         *   public static FileSystem get(URI uri, Configuration conf) throws IOException
         *   public static FileSystem get(URI uri, Configuration conf, String user)
         *   throws IOException
         */
        FileSystem fs = FileSystem.get(URI.create(path), config);
        //??FSDataInputStream,DataInputStream,????
        //open4KB
        try (InputStream in = fs.open(new Path(path))) {
            IOUtils.copy(in, System.out);
        }
    }

}