Example usage for org.apache.hadoop.fs FileSystem getFileStatus

List of usage examples for org.apache.hadoop.fs FileSystem getFileStatus

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. getFileStatus

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getFileStatus.

Prototype

public abstract FileStatus getFileStatus(Path f) throws IOException;

Source Link

Document

Return a file status object that represents the path.

Usage

From source file:com.asakusafw.runtime.directio.hadoop.HadoopDataSourceUtil.java

License:Apache License

/**
 * Searches file/directories by pattern.
 * @param fs target file system/*from  w w  w  . j a v a2  s. co  m*/
 * @param base base path
 * @param pattern search pattern
 * @return found files, or an empty list if not found
 * @throws IOException if failed to search by I/O error
 * @throws IllegalArgumentException if some parameters were {@code null}
 */
public static List<FileStatus> search(FileSystem fs, Path base, FilePattern pattern) throws IOException {
    if (fs == null) {
        throw new IllegalArgumentException("fs must not be null"); //$NON-NLS-1$
    }
    if (base == null) {
        throw new IllegalArgumentException("base must not be null"); //$NON-NLS-1$
    }
    if (pattern == null) {
        throw new IllegalArgumentException("pattern must not be null"); //$NON-NLS-1$
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Start searching for files (path={0}, resourcePattern={1})", //$NON-NLS-1$
                base, pattern));
    }
    List<FileStatus> current = new ArrayList<>(1);
    try {
        FileStatus stat = fs.getFileStatus(base);
        current.add(stat);
    } catch (FileNotFoundException e) {
        return Collections.emptyList();
    }
    int steps = 0;
    LinkedList<Segment> segments = new LinkedList<>(pattern.getSegments());
    while (segments.isEmpty() == false) {
        if (segments.getFirst().isTraverse()) {
            segments.removeFirst();
            current = recursiveStep(fs, current);
        } else {
            List<Path> step = consumeStep(segments);
            current = globStep(fs, current, step);
        }
        steps++;
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format(
                "Finish searching for files (path={0}, resourcePattern={1}, results={2}, steps={3})", //$NON-NLS-1$
                base, pattern, current.size(), steps));
    }
    return current;
}

From source file:com.asakusafw.runtime.directio.hadoop.HadoopDataSourceUtil.java

License:Apache License

private static void move(Counter counter, FileSystem fromFs, Path from, FileSystem toFs, Path to,
        boolean fromLocal) throws IOException {
    if (counter == null) {
        throw new IllegalArgumentException("counter must not be null"); //$NON-NLS-1$
    }/* ww w  . j  a  v a2  s  .  c  o  m*/
    if (fromFs == null) {
        throw new IllegalArgumentException("fromFs must not be null"); //$NON-NLS-1$
    }
    if (from == null) {
        throw new IllegalArgumentException("from must not be null"); //$NON-NLS-1$
    }
    if (toFs == null) {
        throw new IllegalArgumentException("toFs must not be null"); //$NON-NLS-1$
    }
    if (to == null) {
        throw new IllegalArgumentException("to must not be null"); //$NON-NLS-1$
    }
    if (fromLocal && isLocalPath(from) == false) {
        throw new IllegalArgumentException("from must be on local file system"); //$NON-NLS-1$
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Start moving files (from={0}, to={1})", //$NON-NLS-1$
                from, to));
    }
    Path source = fromFs.makeQualified(from);
    Path target = toFs.makeQualified(to);
    List<Path> list = createFileListRelative(counter, fromFs, source);
    if (list.isEmpty()) {
        return;
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Process moving files (from={0}, to={1}, count={2})", //$NON-NLS-1$
                from, to, list.size()));
    }
    Set<Path> directoryCreated = new HashSet<>();
    for (Path path : list) {
        Path sourceFile = new Path(source, path);
        Path targetFile = new Path(target, path);
        if (LOG.isTraceEnabled()) {
            FileStatus stat = fromFs.getFileStatus(sourceFile);
            LOG.trace(MessageFormat.format("Moving file (from={0}, to={1}, size={2})", //$NON-NLS-1$
                    sourceFile, targetFile, stat.getLen()));
        }
        try {
            FileStatus stat = toFs.getFileStatus(targetFile);
            if (LOG.isDebugEnabled()) {
                LOG.debug(MessageFormat.format("Deleting file: {0}", //$NON-NLS-1$
                        targetFile));
            }
            if (FileSystemCompatibility.isDirectory(stat)) {
                toFs.delete(targetFile, true);
            } else {
                toFs.delete(targetFile, false);
            }
        } catch (FileNotFoundException e) {
            Path targetParent = targetFile.getParent();
            if (directoryCreated.contains(targetParent) == false) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug(MessageFormat.format("Creating directory: {0}", //$NON-NLS-1$
                            targetParent));
                }
                toFs.mkdirs(targetParent);
                directoryCreated.add(targetParent);
            }
        }
        counter.add(1);
        if (fromLocal) {
            toFs.moveFromLocalFile(sourceFile, targetFile);
        } else {
            boolean succeed = toFs.rename(sourceFile, targetFile);
            if (succeed == false) {
                throw new IOException(
                        MessageFormat.format("Failed to move file (from={0}, to={1})", sourceFile, targetFile));
            }
        }
        counter.add(1);
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Finish moving files (from={0}, to={1}, count={2})", //$NON-NLS-1$
                from, to, list.size()));
    }
}

From source file:com.asakusafw.runtime.directio.hadoop.HadoopDataSourceUtil.java

License:Apache License

@SuppressWarnings("unchecked")
private static List<Path> createFileListRelative(Counter counter, FileSystem fs, Path source)
        throws IOException {
    assert counter != null;
    assert fs != null;
    assert source != null;
    assert source.isAbsolute();
    URI baseUri = source.toUri();
    FileStatus root;/*from w ww.jav  a2 s  .  c  o m*/
    try {
        root = fs.getFileStatus(source);
    } catch (FileNotFoundException e) {
        LOG.warn(MessageFormat.format("Source path is not found: {0} (May be already moved)", baseUri));
        return Collections.emptyList();
    }
    counter.add(1);
    List<FileStatus> all = recursiveStep(fs, Collections.singletonList(root));
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Source path contains {1} files/directories: {0}", //$NON-NLS-1$
                baseUri, all.size()));
    }
    List<Path> results = new ArrayList<>();
    for (FileStatus stat : all) {
        if (FileSystemCompatibility.isDirectory(stat)) {
            continue;
        }
        Path path = stat.getPath();
        URI uri = path.toUri();
        URI relative = baseUri.relativize(uri);
        if (relative.equals(uri) == false) {
            results.add(new Path(relative));
        } else {
            throw new IOException(MessageFormat.format("Failed to compute relative path: base={0}, target={1}",
                    baseUri, uri));
        }
        counter.add(1);
    }
    Collections.sort(results);
    return results;
}

From source file:com.asakusafw.runtime.directio.hadoop.SequenceFileFormat.java

License:Apache License

@Override
public ModelInput<T> createInput(Class<? extends T> dataType, FileSystem fileSystem, Path path, long offset,
        long fragmentSize, final Counter counter) throws IOException, InterruptedException {
    final long end = offset + fragmentSize;
    final K keyBuffer = createKeyObject();
    final V valueBuffer = createValueObject();
    final SequenceFile.Reader reader;
    try {//from www. j a  v a  2s. co  m
        reader = new SequenceFile.Reader(fileSystem, path, getConf());
    } catch (EOFException e) {
        FileStatus status = fileSystem.getFileStatus(path);
        if (status.getLen() == 0L) {
            LOG.warn(MessageFormat.format("Target sequence file is empty: {0}", path));
            return new ModelInput<T>() {
                @Override
                public boolean readTo(T model) throws IOException {
                    return false;
                }

                @Override
                public void close() throws IOException {
                    return;
                }
            };
        }
        throw e;
    }
    boolean succeed = false;
    try {
        if (offset > reader.getPosition()) {
            reader.sync(offset);
        }
        ModelInput<T> result = new ModelInput<T>() {

            private boolean next = reader.getPosition() < end;

            private long lastPosition = reader.getPosition();

            @Override
            public boolean readTo(T model) throws IOException {
                if (next == false) {
                    return false;
                }
                long current = reader.getPosition();
                @SuppressWarnings("unchecked")
                K key = (K) reader.next(keyBuffer);
                if (key == null || (current >= end && reader.syncSeen())) {
                    next = false;
                    return false;
                } else {
                    reader.getCurrentValue(valueBuffer);
                    SequenceFileFormat.this.copyToModel(keyBuffer, valueBuffer, model);
                    long nextPosition = reader.getPosition();
                    counter.add(nextPosition - lastPosition);
                    lastPosition = nextPosition;
                    return true;
                }
            }

            @Override
            public void close() throws IOException {
                reader.close();
            }
        };
        succeed = true;
        return result;
    } finally {
        if (succeed == false) {
            reader.close();
        }
    }
}

From source file:com.asakusafw.runtime.stage.AbstractCleanupStageClient.java

License:Apache License

@Override
protected int execute(String[] args) throws IOException, InterruptedException {
    Configuration conf = getConf();
    Path path = getPath(conf);//from www. j av a2 s  .c o m
    FileSystem fileSystem = FileSystem.get(path.toUri(), conf);
    String info = MessageFormat.format("batchId={0}, flowId={1}, executionId={2}, operationId={3}, path={4}", //$NON-NLS-1$
            getBatchId(), getFlowId(), getExecutionId(), getOperationId(), path);
    try {
        LOG.info(MessageFormat.format("Searching for cleanup target: {0}", info));
        long start = System.currentTimeMillis();
        if (RuntimeContext.get().isSimulation()) {
            LOG.info(MessageFormat.format(
                    "Skip deleting cleanup target because current execution is in simulation mode: {0}", info));
        } else {
            FileStatus stat = fileSystem.getFileStatus(path);
            if (stat == null) {
                throw new FileNotFoundException(path.toString());
            }
            LOG.info(MessageFormat.format("Start deleting cleanup target: {0}", info));
            if (fileSystem.delete(path, true) == false) {
                throw new IOException("FileSystem.delete() returned false");
            }
        }
        long end = System.currentTimeMillis();
        LOG.info(MessageFormat.format("Finish deleting cleanup target: {0}, elapsed={1}ms", info, end - start));
        return 0;
    } catch (FileNotFoundException e) {
        LOG.warn(MessageFormat.format("Cleanup target is missing: {0}", info));
        return 0;
    } catch (IOException e) {
        LOG.warn(MessageFormat.format("Failed to delete cleanup target: {0}", info), e);
        return 1;
    } finally {
        FileSystem.closeAll();
    }
}

From source file:com.asakusafw.windgate.hadoopfs.ssh.AbstractSshHadoopFsMirrorTest.java

License:Apache License

private void put(FileList.Writer writer, String path, String... contents) throws IOException {
    Configuration conf = new Configuration();
    File temp = folder.newFile(path);
    FileSystem fs = FileSystem.getLocal(conf);
    try (ModelOutput<Text> output = TemporaryStorage.openOutput(conf, Text.class, new Path(temp.toURI()))) {
        for (String content : contents) {
            output.write(new Text(content));
        }/*from   ww w.ja  v  a2  s . c  o  m*/
    }
    FileStatus status = fs.getFileStatus(new Path(temp.toURI()));
    try (FSDataInputStream src = fs.open(status.getPath());
            OutputStream dst = writer.openNext(status.getPath())) {
        byte[] buf = new byte[256];
        while (true) {
            int read = src.read(buf);
            if (read < 0) {
                break;
            }
            dst.write(buf, 0, read);
        }
    }
}

From source file:com.awcoleman.StandaloneJava.AvroCounterByBlock.java

License:Apache License

public AvroCounterByBlock(String inDirStr) throws IOException {

    long numAvroRecords = 0;

    //Get list of input files
    ArrayList<FileStatus> inputFileList = new ArrayList<FileStatus>();

    Configuration conf = new Configuration();
    conf.addResource(new Path("/etc/hadoop/conf/core-site.xml"));
    conf.set("dfs.replication", "1"); //see http://stackoverflow.com/questions/24548699/how-to-append-to-an-hdfs-file-on-an-extremely-small-cluster-3-nodes-or-less

    FileSystem hdfs = null;
    try {/*from ww w  .  ja va 2 s  . co m*/
        hdfs = FileSystem.get(conf);
    } catch (java.io.IOException ioe) {
        System.out.println("Error opening HDFS filesystem. Exiting. Error message: " + ioe.getMessage());
        System.exit(1);
    }
    if (hdfs.getStatus() == null) {
        System.out.println("Unable to contact HDFS filesystem. Exiting.");
        System.exit(1);
    }

    //Check if input dirs/file exists and get file list (even if list of single file)
    Path inPath = new Path(inDirStr);
    if (hdfs.exists(inPath) && hdfs.isFile(inPath)) { //single file
        inputFileList.add(hdfs.getFileStatus(inPath));
    } else if (hdfs.exists(inPath) && hdfs.isDirectory(inPath)) { //dir
        //Get list of input files
        RemoteIterator<LocatedFileStatus> fileStatusListIterator = hdfs.listFiles(inPath, true);
        while (fileStatusListIterator.hasNext()) {
            LocatedFileStatus fileStatus = fileStatusListIterator.next();

            if (fileStatus.isFile() && !fileStatus.getPath().getName().equals("_SUCCESS")) {
                inputFileList.add((FileStatus) fileStatus);
            }
        }
    } else {
        System.out.println("Input directory ( " + inDirStr + " ) not found or is not directory. Exiting.");
        System.exit(1);
    }

    for (FileStatus thisFileStatus : inputFileList) {

        //_SUCCESS files are 0 bytes
        if (thisFileStatus.getLen() == 0) {
            continue;
        }

        DataFileStream<Object> dfs = null;
        FSDataInputStream inStream = hdfs.open(thisFileStatus.getPath());
        GenericDatumReader<Object> reader = new GenericDatumReader<Object>();
        dfs = new DataFileStream<Object>(inStream, reader);

        long thisFileRecords = 0;
        while (dfs.hasNext()) {

            numAvroRecords = numAvroRecords + dfs.getBlockCount();
            thisFileRecords = thisFileRecords + dfs.getBlockCount();

            //System.out.println("Input file "+thisFileStatus.getPath()+" getBlockCount() is "+dfs.getBlockCount()+"." );

            dfs.nextBlock();
        }

        System.out.println("Input file " + thisFileStatus.getPath() + " has " + thisFileRecords + " records.");

        dfs.close();
        inStream.close();

        //TODO test on dir with non-avro file and see what the exception is, catch that and log to output but don't die.
    }

    System.out.println("Input dir/file ( " + inDirStr + " ) has " + inputFileList.size() + " files and "
            + numAvroRecords + " total records.");

}

From source file:com.awcoleman.StandaloneJava.AvroCounterByRecord.java

License:Apache License

public AvroCounterByRecord(String inDirStr) throws IOException {

    long numAvroRecords = 0;

    //Get list of input files
    ArrayList<FileStatus> inputFileList = new ArrayList<FileStatus>();

    Configuration conf = new Configuration();
    conf.addResource(new Path("/etc/hadoop/conf/core-site.xml"));
    conf.set("dfs.replication", "1"); //see http://stackoverflow.com/questions/24548699/how-to-append-to-an-hdfs-file-on-an-extremely-small-cluster-3-nodes-or-less

    FileSystem hdfs = null;
    try {//from w  ww.j  a v a2 s  .  c  om
        hdfs = FileSystem.get(conf);
    } catch (java.io.IOException ioe) {
        System.out.println("Error opening HDFS filesystem. Exiting. Error message: " + ioe.getMessage());
        System.exit(1);
    }
    if (hdfs.getStatus() == null) {
        System.out.println("Unable to contact HDFS filesystem. Exiting.");
        System.exit(1);
    }

    //Check if input dirs/file exists and get file list (even if list of single file)
    Path inPath = new Path(inDirStr);
    if (hdfs.exists(inPath) && hdfs.isFile(inPath)) { //single file
        inputFileList.add(hdfs.getFileStatus(inPath));
    } else if (hdfs.exists(inPath) && hdfs.isDirectory(inPath)) { //dir
        //Get list of input files
        RemoteIterator<LocatedFileStatus> fileStatusListIterator = hdfs.listFiles(inPath, true);
        while (fileStatusListIterator.hasNext()) {
            LocatedFileStatus fileStatus = fileStatusListIterator.next();

            if (fileStatus.isFile() && !fileStatus.getPath().getName().equals("_SUCCESS")) {
                inputFileList.add((FileStatus) fileStatus);
            }
        }
    } else {
        System.out.println("Input directory ( " + inDirStr + " ) not found or is not directory. Exiting.");
        System.exit(1);
    }

    for (FileStatus thisFileStatus : inputFileList) {

        //_SUCCESS files are 0 bytes
        if (thisFileStatus.getLen() == 0) {
            continue;
        }

        DataFileStream<Object> avroStream = null;
        FSDataInputStream inStream = hdfs.open(thisFileStatus.getPath());
        GenericDatumReader<Object> reader = new GenericDatumReader<Object>();
        avroStream = new DataFileStream<Object>(inStream, reader);

        long thisFileRecords = 0;

        while (avroStream.hasNext()) {
            numAvroRecords++;
            thisFileRecords++;
            avroStream.next();
        }
        avroStream.close();
        inStream.close();

        System.out.println("Input file " + thisFileStatus.getPath() + " has " + thisFileRecords + " records.");

        //TODO test on dir with non-avro file and see what the exception is, catch that and log to output but don't die.
    }

    System.out.println("Input dir/file ( " + inDirStr + " ) has " + inputFileList.size() + " files and "
            + numAvroRecords + " total records.");

}

From source file:com.bigdog.hadoop.hdfs.HDFS_Test.java

public byte[] readHDFSFile(String dst) throws Exception {
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(conf);

    // check if the file exists
    Path path = new Path(dst);
    if (fs.exists(path)) {
        FSDataInputStream is = fs.open(path);
        // get the file info to create the buffer
        FileStatus stat = fs.getFileStatus(path);

        // create the buffer
        byte[] buffer = new byte[Integer.parseInt(String.valueOf(stat.getLen()))];
        is.readFully(0, buffer);//w  w  w .  ja  va  2 s .  c o m

        is.close();
        fs.close();

        return buffer;
    } else {
        throw new Exception("the file is not found .");
    }
}

From source file:com.bigjob.Client.java

License:Apache License

/**
 * Main run function for the client//from  ww w.  j a v a2 s.  c o m
 * @return true if application completed successfully
 * @throws IOException
 * @throws YarnException
 */
public boolean run() throws IOException, YarnException {

    LOG.info("Running Client");
    yarnClient.start();

    YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics();
    LOG.info("Got Cluster metric info from ASM (RM)" + ", numNodeManagers="
            + clusterMetrics.getNumNodeManagers());

    List<NodeReport> clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
    LOG.info("Got Cluster node info from ASM");
    for (NodeReport node : clusterNodeReports) {
        LOG.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress"
                + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers"
                + node.getNumContainers());
    }

    QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue);
    LOG.info("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueCurrentCapacity="
            + queueInfo.getCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity()
            + ", queueApplicationCount=" + queueInfo.getApplications().size() + ", queueChildQueueCount="
            + queueInfo.getChildQueues().size());

    List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo();
    for (QueueUserACLInfo aclInfo : listAclInfo) {
        for (QueueACL userAcl : aclInfo.getUserAcls()) {
            LOG.info("User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl="
                    + userAcl.name());
        }
    }

    // Get a new application id
    YarnClientApplication app = yarnClient.createApplication();
    GetNewApplicationResponse appResponse = app.getNewApplicationResponse();
    // TODO get min/max resource capabilities from RM and change memory ask if needed
    // If we do not have min/max, we may not be able to correctly request 
    // the required resources from the RM for the app master
    // Memory ask has to be a multiple of min and less than max. 
    // Dump out information about cluster capability as seen by the resource manager
    int maxMem = appResponse.getMaximumResourceCapability().getMemory();
    LOG.info("Max mem capabililty of resources in this cluster " + maxMem);

    // A resource ask cannot exceed the max. 
    if (amMemory > maxMem) {
        LOG.info("AM memory specified above max threshold of cluster. Using max value." + ", specified="
                + amMemory + ", max=" + maxMem);
        amMemory = maxMem;
    }

    int maxVCores = appResponse.getMaximumResourceCapability().getVirtualCores();
    LOG.info("Max virtual cores capabililty of resources in this cluster " + maxVCores);

    if (amVCores > maxVCores) {
        LOG.info("AM virtual cores specified above max threshold of cluster. " + "Using max value."
                + ", specified=" + amVCores + ", max=" + maxVCores);
        amVCores = maxVCores;
    }

    // set the application name
    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
    ApplicationId appId = appContext.getApplicationId();
    appContext.setApplicationName(appName);

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);

    // set local resources for the application master
    // local files or archives as needed
    // In this scenario, the jar file for the application master is part of the local resources         
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();

    LOG.info("Copy App Master jar from local filesystem and add to local environment");
    // Copy the application master jar to the filesystem 
    // Create a local resource to point to the destination jar path 
    //    if (dfsUrl!=null && dfsUrl.equals("")==false){
    //       conf.set("fs.defaultFS", dfsUrl);
    //    }
    FileSystem fs = FileSystem.get(conf);
    addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.getId(), localResources, null);

    // Set the log4j properties if needed 
    if (!log4jPropFile.isEmpty()) {
        addToLocalResources(fs, log4jPropFile, log4jPath, appId.getId(), localResources, null);
    }

    // The shell script has to be made available on the final container(s)
    // where it will be executed. 
    // To do this, we need to first copy into the filesystem that is visible 
    // to the yarn framework. 
    // We do not need to set this as a local resource for the application 
    // master as the application master does not need it.       
    String hdfsShellScriptLocation = "";
    long hdfsShellScriptLen = 0;
    long hdfsShellScriptTimestamp = 0;
    if (!shellScriptPath.isEmpty()) {
        Path shellSrc = new Path(shellScriptPath);
        String shellPathSuffix = appName + "/" + appId.getId() + "/"
                + (Shell.WINDOWS ? windowBatPath : linuxShellPath);
        Path shellDst = new Path(fs.getHomeDirectory(), shellPathSuffix);
        fs.copyFromLocalFile(false, true, shellSrc, shellDst);
        hdfsShellScriptLocation = shellDst.toUri().toString();
        FileStatus shellFileStatus = fs.getFileStatus(shellDst);
        hdfsShellScriptLen = shellFileStatus.getLen();
        hdfsShellScriptTimestamp = shellFileStatus.getModificationTime();
    }

    if (!shellCommand.isEmpty()) {
        addToLocalResources(fs, null, shellCommandPath, appId.getId(), localResources, shellCommand);
    }

    if (shellArgs.length > 0) {
        addToLocalResources(fs, null, shellArgsPath, appId.getId(), localResources,
                StringUtils.join(shellArgs, " "));
    }
    // Set local resource info into app master container launch context
    amContainer.setLocalResources(localResources);

    // Set the necessary security tokens as needed
    //amContainer.setContainerTokens(containerToken);

    // Set the env variables to be setup in the env where the application master will be run
    LOG.info("Set the environment for the application master");
    Map<String, String> env = new HashMap<String, String>();

    // put location of shell script into env
    // using the env info, the application master will create the correct local resource for the 
    // eventual containers that will be launched to execute the shell scripts
    env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION, hdfsShellScriptLocation);
    env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP, Long.toString(hdfsShellScriptTimestamp));
    env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN, Long.toString(hdfsShellScriptLen));

    // Add AppMaster.jar location to classpath       
    // At some point we should not be required to add 
    // the hadoop specific classpaths to the env. 
    // It should be provided out of the box. 
    // For now setting all required classpaths including
    // the classpath to "." for the application jar
    StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$()).append(File.pathSeparatorChar)
            .append("./*");
    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        classPathEnv.append(File.pathSeparatorChar);
        classPathEnv.append(c.trim());
    }
    classPathEnv.append(File.pathSeparatorChar).append("./log4j.properties");

    // add the runtime classpath needed for tests to work
    if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
        classPathEnv.append(':');
        classPathEnv.append(System.getProperty("java.class.path"));
    }

    env.put("CLASSPATH", classPathEnv.toString());

    amContainer.setEnvironment(env);

    // Set the necessary command to execute the application master 
    Vector<CharSequence> vargs = new Vector<CharSequence>(30);

    // Set java executable command 
    LOG.info("Setting up app master command");
    vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
    // Set Xmx based on am memory size
    vargs.add("-Xmx" + amMemory + "m");
    // Set class name 
    vargs.add(appMasterMainClass);
    // Set params for Application Master
    vargs.add("--container_memory " + String.valueOf(containerMemory));
    vargs.add("--container_vcores " + String.valueOf(containerVirtualCores));
    vargs.add("--num_containers " + String.valueOf(numContainers));
    vargs.add("--priority " + String.valueOf(shellCmdPriority));

    for (Map.Entry<String, String> entry : shellEnv.entrySet()) {
        vargs.add("--shell_env " + entry.getKey() + "=" + entry.getValue());
    }
    if (debugFlag) {
        vargs.add("--debug");
    }

    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout");
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr");

    // Get final commmand
    StringBuilder command = new StringBuilder();
    for (CharSequence str : vargs) {
        command.append(str).append(" ");
    }

    LOG.info("Completed setting up app master command " + command.toString());
    List<String> commands = new ArrayList<String>();
    commands.add(command.toString());
    amContainer.setCommands(commands);

    // Set up resource type requirements
    // For now, both memory and vcores are supported, so we set memory and 
    // vcores requirements
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(amMemory);
    capability.setVirtualCores(amVCores);
    appContext.setResource(capability);

    // Service data is a binary blob that can be passed to the application
    // Not needed in this scenario
    // amContainer.setServiceData(serviceData);

    // Setup security tokens
    if (UserGroupInformation.isSecurityEnabled()) {
        Credentials credentials = new Credentials();
        String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL);
        if (tokenRenewer == null || tokenRenewer.length() == 0) {
            throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer");
        }

        // For now, only getting tokens for the default file-system.
        final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials);
        if (tokens != null) {
            for (Token<?> token : tokens) {
                LOG.info("Got dt for " + fs.getUri() + "; " + token);
            }
        }
        DataOutputBuffer dob = new DataOutputBuffer();
        credentials.writeTokenStorageToStream(dob);
        ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
        amContainer.setTokens(fsTokens);
    }

    appContext.setAMContainerSpec(amContainer);

    // Set the priority for the application master
    Priority pri = Records.newRecord(Priority.class);
    // TODO - what is the range for priority? how to decide? 
    pri.setPriority(amPriority);
    appContext.setPriority(pri);

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue(amQueue);

    // Submit the application to the applications manager
    // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest);
    // Ignore the response as either a valid response object is returned on success 
    // or an exception thrown to denote some form of a failure
    LOG.info("Submitting application to ASM");

    yarnClient.submitApplication(appContext);

    // TODO
    // Try submitting the same request again
    // app submission failure?

    // Monitor the application
    //return monitorApplication(appId);
    System.out.println("ApplicationId:" + appId);
    return true;
}