Example usage for org.apache.hadoop.fs FileSystem getConf

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getConf.

Prototype

@Override
    public Configuration getConf()

Source Link

Usage

From source file:tl.lin.data.util.SequenceFileUtils.java

License:Apache License

/**
 * Reads key-value pairs from a SequenceFile, up to a maximum number.
 *
 * @param path path to file/*from  ww w  . j  av  a 2s .c  om*/
 * @param max maximum of key-value pairs to read
 * @return list of key-value pairs
 */
@SuppressWarnings("unchecked")
public static <K extends Writable, V extends Writable> List<PairOfWritables<K, V>> readFile(Path path,
        FileSystem fs, int max) throws IOException {
    List<PairOfWritables<K, V>> list = new ArrayList<PairOfWritables<K, V>>();

    try {
        int k = 0;
        SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, fs.getConf());

        K key;
        V value;

        key = (K) reader.getKeyClass().newInstance();
        value = (V) reader.getValueClass().newInstance();

        while (reader.next(key, value)) {
            k++;
            list.add(new PairOfWritables<K, V>(key, value));
            if (k >= max) {
                break;
            }

            // Create new objects, because the key, value gets reused
            key = (K) reader.getKeyClass().newInstance();
            value = (V) reader.getValueClass().newInstance();
        }
        reader.close();
    } catch (IllegalAccessException e) {
        throw new RuntimeException("Error reading SequenceFile: " + e);
    } catch (InstantiationException e) {
        throw new RuntimeException("Error reading SequenceFile: " + e);
    }

    return list;
}