org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageDecompressor.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageDecompressor.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hdfs.tools.offlineImageViewer;

import java.io.BufferedOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.PositionTrackingInputStream;
import org.apache.hadoop.hdfs.server.namenode.FSImage;
import org.apache.hadoop.hdfs.server.namenode.FSImageCompression;
import org.apache.hadoop.io.BufferedByteInputStream;

/**
 * OfflineImageViewer to dump the contents of an Hadoop image file to XML or the
 * console. Main entry point into utility, either via the command line or
 * programatically.
 */
public class OfflineImageDecompressor {

    private final static String usage = "Usage: bin/hdfs oid -i INPUTFILE -o OUTPUTFILE\n"
            + "Offline Image Decompressor\n" + "The oid utility will attempt to decompress image files.\n"
            + "The tool works offline and does not require a running cluster in\n"
            + "order to process an image file.\n" + "Required command line arguments:\n"
            + "-i,--inputFile <arg>   FSImage file to process.\n"
            + "-o,--outputFile <arg>  Name of output file. If the specified\n"
            + "                       file exists, it will be overwritten.\n";

    private final String inputFile;
    private final String outputFile;
    private int lastProgress = 0;

    public OfflineImageDecompressor(String inputFile, String outputFile) {
        this.inputFile = inputFile;
        this.outputFile = outputFile;
    }

    /**
     * Process image file.
     */
    private void go() throws IOException {
        long start = System.currentTimeMillis();
        System.out.println("Decompressing image file: " + inputFile + " to " + outputFile);
        DataInputStream in = null;
        DataOutputStream out = null;

        try {
            // setup in
            PositionTrackingInputStream ptis = new PositionTrackingInputStream(
                    new FileInputStream(new File(inputFile)));
            in = new DataInputStream(ptis);

            // read header information
            int imgVersion = in.readInt();
            if (!LayoutVersion.supports(Feature.FSIMAGE_COMPRESSION, imgVersion)) {
                System.out.println("Image is not compressed. No output will be produced.");
                return;
            }
            int namespaceId = in.readInt();
            long numFiles = in.readLong();
            long genstamp = in.readLong();

            long imgTxId = -1;
            if (LayoutVersion.supports(Feature.STORED_TXIDS, imgVersion)) {
                imgTxId = in.readLong();
            }
            FSImageCompression compression = FSImageCompression.readCompressionHeader(new Configuration(), in);
            if (compression.isNoOpCompression()) {
                System.out.println("Image is not compressed. No output will be produced.");
                return;
            }
            in = BufferedByteInputStream.wrapInputStream(compression.unwrapInputStream(in),
                    FSImage.LOAD_SAVE_BUFFER_SIZE, FSImage.LOAD_SAVE_CHUNK_SIZE);
            System.out.println("Starting decompression.");

            // setup output
            out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(outputFile)));

            // write back the uncompressed information
            out.writeInt(imgVersion);
            out.writeInt(namespaceId);
            out.writeLong(numFiles);
            out.writeLong(genstamp);
            if (LayoutVersion.supports(Feature.STORED_TXIDS, imgVersion)) {
                out.writeLong(imgTxId);
            }
            // no compression
            out.writeBoolean(false);

            // copy the data
            long size = new File(inputFile).length();
            // read in 1MB chunks
            byte[] block = new byte[1024 * 1024];
            while (true) {
                int bytesRead = in.read(block);
                if (bytesRead <= 0)
                    break;
                out.write(block, 0, bytesRead);
                printProgress(ptis.getPos(), size);
            }

            out.close();

            long stop = System.currentTimeMillis();
            System.out.println("Input file : " + inputFile + " size: " + size);
            System.out.println("Output file: " + outputFile + " size: " + new File(outputFile).length());
            System.out.println("Decompression completed in " + (stop - start) + " ms.");
        } finally {
            if (in != null)
                in.close();
            if (out != null)
                out.close();
        }
    }

    /**
     * Print the progress.
     */
    private void printProgress(long read, long size) {
        int progress = Math.min(100, (int) ((100 * read) / size));
        if (progress > lastProgress) {
            lastProgress = progress;
            System.out.println("Completed " + lastProgress + " % ");
        }
    }

    /**
     * Build command-line options and descriptions
     */
    public static Options buildOptions() {
        Options options = new Options();

        // Build in/output file arguments, which are required, but there is no
        // addOption method that can specify this
        OptionBuilder.isRequired();
        OptionBuilder.hasArgs();
        OptionBuilder.withLongOpt("outputFile");
        options.addOption(OptionBuilder.create("o"));

        OptionBuilder.isRequired();
        OptionBuilder.hasArgs();
        OptionBuilder.withLongOpt("inputFile");
        options.addOption(OptionBuilder.create("i"));

        options.addOption("h", "help", false, "");
        return options;
    }

    /**
     * Entry point to command-line-driven operation. User may specify options and
     * start fsimage viewer from the command line. Program will process image file
     * and exit cleanly or, if an error is encountered, inform user and exit.
     * 
     * @param args
     *          Command line options
     * @throws IOException
     */
    public static void main(String[] args) throws IOException {
        Options options = buildOptions();
        if (args.length == 0) {
            printUsage();
            return;
        }

        CommandLineParser parser = new PosixParser();
        CommandLine cmd;

        try {
            cmd = parser.parse(options, args);
        } catch (ParseException e) {
            System.out.println("Error parsing command-line options: ");
            printUsage();
            return;
        }

        if (cmd.hasOption("h")) { // print help and exit
            printUsage();
            return;
        }

        String inputFile = cmd.getOptionValue("i");
        String outputFile = cmd.getOptionValue("o");

        try {
            OfflineImageDecompressor d = new OfflineImageDecompressor(inputFile, outputFile);
            d.go();
        } catch (EOFException e) {
            System.err.println("Input file ended unexpectedly.  Exiting");
            System.exit(255);
        } catch (IOException e) {
            System.err.println("Encountered exception.  Exiting: " + e.getMessage());
            System.exit(1);
        }
    }

    /**
     * Print application usage instructions.
     */
    private static void printUsage() {
        System.out.println(usage);
    }
}