com.netflix.bdp.s3mper.cli.FileSystemVerifyCommand.java Source code

Introduction

Here is the source code for com.netflix.bdp.s3mper.cli.FileSystemVerifyCommand.java

Source

/*
 *
 *  Copyright 2013 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */

package com.netflix.bdp.s3mper.cli;

import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3Client;
import com.netflix.bdp.s3mper.common.PathUtil;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.Logger;
import org.kohsuke.args4j.CmdLineException;
import org.kohsuke.args4j.CmdLineParser;
import org.kohsuke.args4j.Option;
import org.kohsuke.args4j.OptionHandlerFilter;

/**
 *
 * @author dweeks
 */
public class FileSystemVerifyCommand extends Command {
    private static final Logger log = Logger.getLogger(FileSystemVerifyCommand.class.getName());

    @Option(name = "-t", usage = "Number of threads")
    private int threads = 10;

    @Option(name = "-f", usage = "Number of threads")
    private String file = null;

    private AmazonS3Client s3;

    @Override
    public void execute(Configuration conf, String[] args) throws Exception {
        CmdLineParser parser = new CmdLineParser(this);

        String keyId = conf.get("fs.s3n.awsAccessKeyId");
        String keySecret = conf.get("fs.s3n.awsSecretAccessKey");

        s3 = new AmazonS3Client(new BasicAWSCredentials(keyId, keySecret));

        try {
            parser.parseArgument(args);

            ExecutorService executor = Executors.newFixedThreadPool(threads);
            List<Future> futures = new ArrayList<Future>();

            BufferedReader fin = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF8"));

            try {
                for (String line = fin.readLine(); line != null; line = fin.readLine()) {
                    futures.add(executor.submit(new FileCheckTask(new Path(line.trim()))));
                }
            } finally {
                fin.close();
            }

            for (Future f : futures) {
                f.get();
            }

            executor.shutdown();
        } catch (CmdLineException e) {
            System.err.println(e.getMessage());
            System.err.println("s3mper fs verify [options]");
            // print the list of available options
            parser.printUsage(System.err);
            System.err.println();

            System.err.println(" Example: s3mper fs verify " + parser.printExample(OptionHandlerFilter.ALL));
        }
    }

    private class FileCheckTask implements Callable<Boolean> {
        private Path path;

        public FileCheckTask(Path path) {
            this.path = path;
        }

        @Override
        public Boolean call() throws Exception {
            boolean exists = false;

            FileSystem fs = FileSystem.get(path.toUri(), new Configuration());

            try {
                exists = fs.exists(path);
            } catch (Exception e) {
            }

            System.out.println((exists ? "[EXISTS ] " : "[MISSING] ") + path.toUri());

            return exists;
        }
    }

}