org.apache.hadoop.yarn.client.cli.TestLogsCLI.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.hadoop.yarn.client.cli.TestLogsCLI.java

Source

/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements.  See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership.  The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License.  You may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.hadoop.yarn.client.cli;

import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;

import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat;
import org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest;
import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils;
import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers;
import org.codehaus.jettison.json.JSONObject;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;

public class TestLogsCLI {

    ByteArrayOutputStream sysOutStream;
    private PrintStream sysOut;

    ByteArrayOutputStream sysErrStream;
    private PrintStream sysErr;

    @Before
    public void setUp() {
        sysOutStream = new ByteArrayOutputStream();
        sysOut = new PrintStream(sysOutStream);
        System.setOut(sysOut);

        sysErrStream = new ByteArrayOutputStream();
        sysErr = new PrintStream(sysErrStream);
        System.setErr(sysErr);
    }

    @Test(timeout = 5000l)
    public void testFailResultCodes() throws Exception {
        Configuration conf = new YarnConfiguration();
        conf.setClass("fs.file.impl", LocalFileSystem.class, FileSystem.class);
        LogCLIHelpers cliHelper = new LogCLIHelpers();
        cliHelper.setConf(conf);
        YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED,
                UserGroupInformation.getCurrentUser().getShortUserName());
        LogsCLI dumper = new LogsCLIForTest(mockYarnClient);
        dumper.setConf(conf);

        // verify dumping a non-existent application's logs returns a failure code
        int exitCode = dumper.run(new String[] { "-applicationId", "application_0_0" });
        assertTrue("Should return an error code", exitCode != 0);

        // verify dumping a non-existent container log is a failure code
        exitCode = cliHelper.dumpAContainersLogs("application_0_0", "container_0_0", "nonexistentnode:1234",
                "nobody");
        assertTrue("Should return an error code", exitCode != 0);
    }

    @Test(timeout = 5000l)
    public void testInvalidApplicationId() throws Exception {
        Configuration conf = new YarnConfiguration();
        YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED,
                UserGroupInformation.getCurrentUser().getShortUserName());
        LogsCLI cli = new LogsCLIForTest(mockYarnClient);
        cli.setConf(conf);

        int exitCode = cli.run(new String[] { "-applicationId", "not_an_app_id" });
        assertTrue(exitCode == -1);
        assertTrue(sysErrStream.toString().startsWith("Invalid ApplicationId specified"));
    }

    @Test(timeout = 5000L)
    public void testInvalidAMContainerId() throws Exception {
        Configuration conf = new YarnConfiguration();
        conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true);
        YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED,
                UserGroupInformation.getCurrentUser().getShortUserName());
        LogsCLI cli = spy(new LogsCLIForTest(mockYarnClient));
        List<JSONObject> list = Arrays.asList(new JSONObject());
        doReturn(list).when(cli).getAMContainerInfoForRMWebService(any(Configuration.class), any(String.class));
        cli.setConf(conf);

        int exitCode = cli.run(new String[] { "-applicationId", "application_1465862913885_0027", "-am", "1000" });
        assertTrue(exitCode == -1);
        assertTrue(sysErrStream.toString().contains("exceeds the number of AM containers"));
    }

    @Test(timeout = 5000l)
    public void testUnknownApplicationId() throws Exception {
        Configuration conf = new YarnConfiguration();
        YarnClient mockYarnClient = createMockYarnClientUnknownApp();
        LogsCLI cli = new LogsCLIForTest(mockYarnClient);
        cli.setConf(conf);

        int exitCode = cli.run(new String[] { "-applicationId", ApplicationId.newInstance(1, 1).toString() });

        // Error since no logs present for the app.
        assertTrue(exitCode != 0);
        assertTrue(sysErrStream.toString().startsWith("Unable to get ApplicationState"));
    }

    @Test(timeout = 5000l)
    public void testHelpMessage() throws Exception {
        Configuration conf = new YarnConfiguration();
        YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED,
                UserGroupInformation.getCurrentUser().getShortUserName());
        LogsCLI dumper = new LogsCLIForTest(mockYarnClient);
        dumper.setConf(conf);

        int exitCode = dumper.run(new String[] {});
        assertTrue(exitCode == -1);
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        PrintWriter pw = new PrintWriter(baos);
        pw.println("Retrieve logs for YARN applications.");
        pw.println("usage: yarn logs -applicationId <application ID> [OPTIONS]");
        pw.println();
        pw.println("general options are:");
        pw.println(" -am <AM Containers>                     Prints the AM Container logs for");
        pw.println("                                         this application. Specify");
        pw.println("                                         comma-separated value to get logs");
        pw.println("                                         for related AM Container. For");
        pw.println("                                         example, If we specify -am 1,2,");
        pw.println("                                         we will get the logs for the");
        pw.println("                                         first AM Container as well as the");
        pw.println("                                         second AM Container. To get logs");
        pw.println("                                         for all AM Containers, use -am");
        pw.println("                                         ALL. To get logs for the latest");
        pw.println("                                         AM Container, use -am -1. By");
        pw.println("                                         default, it will print all");
        pw.println("                                         available logs. Work with");
        pw.println("                                         -log_files to get only specific");
        pw.println("                                         logs.");
        pw.println(" -appOwner <Application Owner>           AppOwner (assumed to be current");
        pw.println("                                         user if not specified)");
        pw.println(" -containerId <Container ID>             ContainerId. By default, it will");
        pw.println("                                         print all available logs. Work");
        pw.println("                                         with -log_files to get only");
        pw.println("                                         specific logs. If specified, the");
        pw.println("                                         applicationId can be omitted");
        pw.println(" -help                                   Displays help for all commands.");
        pw.println(" -list_nodes                             Show the list of nodes that");
        pw.println("                                         successfully aggregated logs.");
        pw.println("                                         This option can only be used with");
        pw.println("                                         finished applications.");
        pw.println(" -log_files <Log File Name>              Specify comma-separated value to");
        pw.println("                                         get exact matched log files. Use");
        pw.println("                                         \"ALL\" or \"*\" to fetch all the log");
        pw.println("                                         files for the container.");
        pw.println(" -log_files_pattern <Log File Pattern>   Specify comma-separated value to");
        pw.println("                                         get matched log files by using");
        pw.println("                                         java regex. Use \".*\" to fetch all");
        pw.println("                                         the log files for the container.");
        pw.println(" -nodeAddress <Node Address>             NodeAddress in the format");
        pw.println("                                         nodename:port");
        pw.println(" -out <Local Directory>                  Local directory for storing");
        pw.println("                                         individual container logs. The");
        pw.println("                                         container logs will be stored");
        pw.println("                                         based on the node the container");
        pw.println("                                         ran on.");
        pw.println(" -show_application_log_info              Show the containerIds which");
        pw.println("                                         belong to the specific");
        pw.println("                                         Application. You can combine this");
        pw.println("                                         with --nodeAddress to get");
        pw.println("                                         containerIds for all the");
        pw.println("                                         containers on the specific");
        pw.println("                                         NodeManager.");
        pw.println(" -show_container_log_info                Show the container log metadata,");
        pw.println("                                         including log-file names, the");
        pw.println("                                         size of the log files. You can");
        pw.println("                                         combine this with --containerId");
        pw.println("                                         to get log metadata for the");
        pw.println("                                         specific container, or with");
        pw.println("                                         --nodeAddress to get log metadata");
        pw.println("                                         for all the containers on the");
        pw.println("                                         specific NodeManager.");
        pw.println(" -size <size>                            Prints the log file's first 'n'");
        pw.println("                                         bytes or the last 'n' bytes. Use");
        pw.println("                                         negative values as bytes to read");
        pw.println("                                         from the end and positive values");
        pw.println("                                         as bytes to read from the");
        pw.println("                                         beginning.");
        pw.close();
        String appReportStr = baos.toString("UTF-8");
        Assert.assertEquals(appReportStr, sysOutStream.toString());
    }

    @Test(timeout = 15000)
    public void testFetchFinishedApplictionLogs() throws Exception {
        String remoteLogRootDir = "target/logs/";
        Configuration configuration = new Configuration();
        configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
        configuration.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
        configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
        configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
        FileSystem fs = FileSystem.get(configuration);

        UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
        ApplicationId appId = ApplicationId.newInstance(0, 1);
        ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
        ContainerId containerId0 = ContainerId.newContainerId(appAttemptId, 0);
        ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1);
        ContainerId containerId2 = ContainerId.newContainerId(appAttemptId, 2);
        ContainerId containerId3 = ContainerId.newContainerId(appAttemptId, 3);
        final NodeId nodeId = NodeId.newInstance("localhost", 1234);

        // create local logs
        String rootLogDir = "target/LocalLogs";
        Path rootLogDirPath = new Path(rootLogDir);
        if (fs.exists(rootLogDirPath)) {
            fs.delete(rootLogDirPath, true);
        }
        assertTrue(fs.mkdirs(rootLogDirPath));

        Path appLogsDir = new Path(rootLogDirPath, appId.toString());
        if (fs.exists(appLogsDir)) {
            fs.delete(appLogsDir, true);
        }
        assertTrue(fs.mkdirs(appLogsDir));

        List<String> rootLogDirs = Arrays.asList(rootLogDir);

        List<String> logTypes = new ArrayList<String>();
        logTypes.add("syslog");
        // create container logs in localLogDir
        createContainerLogInLocalDir(appLogsDir, containerId1, fs, logTypes);
        createContainerLogInLocalDir(appLogsDir, containerId2, fs, logTypes);

        // create two logs for container3 in localLogDir
        logTypes.add("stdout");
        logTypes.add("stdout1234");
        createContainerLogInLocalDir(appLogsDir, containerId3, fs, logTypes);

        Path path = new Path(remoteLogRootDir + ugi.getShortUserName() + "/logs/application_0_0001");
        if (fs.exists(path)) {
            fs.delete(path, true);
        }
        assertTrue(fs.mkdirs(path));

        // upload container logs into remote directory
        // the first two logs is empty. When we try to read first two logs,
        // we will meet EOF exception, but it will not impact other logs.
        // Other logs should be read successfully.
        uploadEmptyContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId0, path, fs);
        uploadEmptyContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId1, path, fs);
        uploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId1, path, fs);
        uploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId2, path, fs);
        uploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId, containerId3, path, fs);

        YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED, ugi.getShortUserName());
        LogsCLI cli = new LogsCLIForTest(mockYarnClient) {
            @Override
            public ContainerReport getContainerReport(String containerIdStr) throws YarnException, IOException {
                ContainerReport mockReport = mock(ContainerReport.class);
                doReturn(nodeId).when(mockReport).getAssignedNode();
                doReturn("http://localhost:2345").when(mockReport).getNodeHttpAddress();
                return mockReport;
            }
        };
        cli.setConf(configuration);

        int exitCode = cli.run(new String[] { "-applicationId", appId.toString() });
        assertTrue(exitCode == 0);
        assertTrue(sysOutStream.toString().contains(logMessage(containerId1, "syslog")));
        assertTrue(sysOutStream.toString().contains(logMessage(containerId2, "syslog")));
        assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "syslog")));
        assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout")));
        assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout1234")));
        sysOutStream.reset();

        exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-log_files_pattern", ".*" });
        assertTrue(exitCode == 0);
        assertTrue(sysOutStream.toString().contains(logMessage(containerId1, "syslog")));
        assertTrue(sysOutStream.toString().contains(logMessage(containerId2, "syslog")));
        assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "syslog")));
        assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout")));
        assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout1234")));
        sysOutStream.reset();

        exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-log_files", "*" });
        assertTrue(exitCode == 0);
        assertTrue(sysOutStream.toString().contains(logMessage(containerId1, "syslog")));
        assertTrue(sysOutStream.toString().contains(logMessage(containerId2, "syslog")));
        assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "syslog")));
        assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout")));
        assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout1234")));
        int fullSize = sysOutStream.toByteArray().length;
        sysOutStream.reset();

        exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-log_files", "stdout" });
        assertTrue(exitCode == 0);
        assertFalse(sysOutStream.toString().contains(logMessage(containerId1, "syslog")));
        assertFalse(sysOutStream.toString().contains(logMessage(containerId2, "syslog")));
        assertFalse(sysOutStream.toString().contains(logMessage(containerId3, "syslog")));
        assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout")));
        assertFalse(sysOutStream.toString().contains(logMessage(containerId3, "stdout1234")));
        sysOutStream.reset();

        exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-log_files_pattern", "std*" });
        assertTrue(exitCode == 0);
        assertFalse(sysOutStream.toString().contains(logMessage(containerId1, "syslog")));
        assertFalse(sysOutStream.toString().contains(logMessage(containerId2, "syslog")));
        assertFalse(sysOutStream.toString().contains(logMessage(containerId3, "syslog")));
        assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout")));
        assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout1234")));
        sysOutStream.reset();

        exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-log_files", "123" });
        assertTrue(exitCode == -1);
        assertTrue(sysErrStream.toString().contains("Can not find any log file matching the pattern: [123] "
                + "for the application: " + appId.toString()));
        sysErrStream.reset();

        // specify the bytes which is larger than the actual file size,
        // we would get the full logs
        exitCode = cli
                .run(new String[] { "-applicationId", appId.toString(), "-log_files", "*", "-size", "10000" });
        assertTrue(exitCode == 0);
        assertTrue(sysOutStream.toByteArray().length == fullSize);
        sysOutStream.reset();

        // uploaded two logs for container1. The first log is empty.
        // The second one is not empty.
        // We can still successfully read logs for container1.
        exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-nodeAddress", nodeId.toString(),
                "-containerId", containerId1.toString() });
        assertTrue(exitCode == 0);
        assertTrue(sysOutStream.toString().contains(logMessage(containerId1, "syslog")));
        assertTrue(sysOutStream.toString().contains("Log Upload Time"));
        assertTrue(!sysOutStream.toString()
                .contains("Logs for container " + containerId1.toString() + " are not present in this log-file."));
        sysOutStream.reset();

        exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-containerId",
                containerId3.toString(), "-log_files", "123" });
        assertTrue(exitCode == -1);
        assertTrue(sysErrStream.toString().contains("Can not find any log file matching the pattern: [123] "
                + "for the container: " + containerId3 + " within the application: " + appId.toString()));
        sysErrStream.reset();

        exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-containerId",
                containerId3.toString(), "-log_files", "stdout" });
        assertTrue(exitCode == 0);
        int fullContextSize = sysOutStream.toByteArray().length;
        String fullContext = sysOutStream.toString();
        sysOutStream.reset();

        String logMessage = logMessage(containerId3, "stdout");
        int fileContentSize = logMessage.getBytes().length;
        int tailContentSize = "\nEnd of LogType:stdout\n\n".getBytes().length;

        // specify how many bytes we should get from logs
        // specify a position number, it would get the first n bytes from
        // container log
        exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-containerId",
                containerId3.toString(), "-log_files", "stdout", "-size", "5" });
        assertTrue(exitCode == 0);
        Assert.assertEquals(new String(logMessage.getBytes(), 0, 5),
                new String(sysOutStream.toByteArray(), (fullContextSize - fileContentSize - tailContentSize), 5));
        sysOutStream.reset();

        // specify a negative number, it would get the last n bytes from
        // container log
        exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-containerId",
                containerId3.toString(), "-log_files", "stdout", "-size", "-5" });
        assertTrue(exitCode == 0);
        Assert.assertEquals(new String(logMessage.getBytes(), logMessage.getBytes().length - 5, 5),
                new String(sysOutStream.toByteArray(), (fullContextSize - fileContentSize - tailContentSize), 5));
        sysOutStream.reset();

        long negative = (fullContextSize + 1000) * (-1);
        exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-containerId",
                containerId3.toString(), "-log_files", "stdout", "-size", Long.toString(negative) });
        assertTrue(exitCode == 0);
        Assert.assertEquals(fullContext, sysOutStream.toString());
        sysOutStream.reset();

        // Uploaded the empty log for container0.
        // We should see the message showing the log for container0
        // are not present.
        exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-nodeAddress", nodeId.toString(),
                "-containerId", containerId0.toString() });
        assertTrue(exitCode == -1);
        assertTrue(sysErrStream.toString()
                .contains("Logs for container " + containerId0.toString() + " are not present in this log-file."));
        sysErrStream.reset();

        // uploaded two logs for container3. The first log is named as syslog.
        // The second one is named as stdout.
        exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-nodeAddress", nodeId.toString(),
                "-containerId", containerId3.toString() });
        assertTrue(exitCode == 0);
        assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "syslog")));
        assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout")));
        sysOutStream.reset();

        // set -log_files option as stdout
        // should only print log with the name as stdout
        exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-nodeAddress", nodeId.toString(),
                "-containerId", containerId3.toString(), "-log_files", "stdout" });
        assertTrue(exitCode == 0);
        assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout")));
        assertTrue(!sysOutStream.toString().contains(logMessage(containerId3, "syslog")));
        sysOutStream.reset();

        YarnClient mockYarnClientWithException = createMockYarnClientWithException();
        cli = new LogsCLIForTest(mockYarnClientWithException);
        cli.setConf(configuration);

        exitCode = cli
                .run(new String[] { "-applicationId", appId.toString(), "-containerId", containerId3.toString() });
        assertTrue(exitCode == 0);
        assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "syslog")));
        assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout")));
        assertTrue(sysOutStream.toString()
                .contains(containerId3 + " on " + LogAggregationUtils.getNodeString(nodeId)));
        sysOutStream.reset();

        // The same should also work without the applicationId
        exitCode = cli.run(new String[] { "-containerId", containerId3.toString() });
        assertTrue(exitCode == 0);
        assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "syslog")));
        assertTrue(sysOutStream.toString().contains(logMessage(containerId3, "stdout")));
        assertTrue(sysOutStream.toString()
                .contains(containerId3 + " on " + LogAggregationUtils.getNodeString(nodeId)));
        sysOutStream.reset();

        exitCode = cli.run(new String[] { "-containerId", "invalid_container" });
        assertTrue(exitCode == -1);
        assertTrue(sysErrStream.toString().contains("Invalid ContainerId specified"));
        sysErrStream.reset();

        fs.delete(new Path(remoteLogRootDir), true);
        fs.delete(new Path(rootLogDir), true);
    }

    @Test(timeout = 5000)
    public void testGetRunningContainerLogs() throws Exception {
        UserGroupInformation ugi = UserGroupInformation.getCurrentUser();

        NodeId nodeId = NodeId.newInstance("localhost", 1234);
        ApplicationId appId = ApplicationId.newInstance(0, 1);
        ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);

        // Create a mock ApplicationAttempt Report
        ApplicationAttemptReport mockAttemptReport = mock(ApplicationAttemptReport.class);
        doReturn(appAttemptId).when(mockAttemptReport).getApplicationAttemptId();
        List<ApplicationAttemptReport> attemptReports = Arrays.asList(mockAttemptReport);

        // Create one mock containerReport
        ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1);
        ContainerReport mockContainerReport1 = mock(ContainerReport.class);
        doReturn(containerId1).when(mockContainerReport1).getContainerId();
        doReturn(nodeId).when(mockContainerReport1).getAssignedNode();
        doReturn("http://localhost:2345").when(mockContainerReport1).getNodeHttpAddress();
        doReturn(ContainerState.RUNNING).when(mockContainerReport1).getContainerState();
        List<ContainerReport> containerReports = Arrays.asList(mockContainerReport1);

        // Mock the YarnClient, and it would report the previous created
        // mockAttemptReport and previous two created mockContainerReports
        YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.RUNNING, ugi.getShortUserName(), true,
                attemptReports, containerReports);
        doReturn(mockContainerReport1).when(mockYarnClient).getContainerReport(any(ContainerId.class));

        // create local logs
        Configuration configuration = new Configuration();
        FileSystem fs = FileSystem.get(configuration);
        String rootLogDir = "target/LocalLogs";
        Path rootLogDirPath = new Path(rootLogDir);
        if (fs.exists(rootLogDirPath)) {
            fs.delete(rootLogDirPath, true);
        }
        assertTrue(fs.mkdirs(rootLogDirPath));

        Path appLogsDir = new Path(rootLogDirPath, appId.toString());
        if (fs.exists(appLogsDir)) {
            fs.delete(appLogsDir, true);
        }
        assertTrue(fs.mkdirs(appLogsDir));

        String fileName = "syslog";
        List<String> logTypes = new ArrayList<String>();
        logTypes.add(fileName);
        // create container logs in localLogDir
        createContainerLogInLocalDir(appLogsDir, containerId1, fs, logTypes);

        Path containerDirPath = new Path(appLogsDir, containerId1.toString());
        Path logPath = new Path(containerDirPath, fileName);
        File logFile = new File(logPath.toString());
        final FileInputStream fis = new FileInputStream(logFile);

        try {
            LogsCLI cli = spy(new LogsCLIForTest(mockYarnClient));
            Set<String> logsSet = new HashSet<String>();
            logsSet.add(fileName);
            doReturn(logsSet).when(cli).getMatchedContainerLogFiles(any(ContainerLogsRequest.class), anyBoolean());
            ClientResponse mockReponse = mock(ClientResponse.class);
            doReturn(ClientResponse.Status.OK).when(mockReponse).getClientResponseStatus();
            doReturn(fis).when(mockReponse).getEntityInputStream();
            doReturn(mockReponse).when(cli).getResponeFromNMWebService(any(Configuration.class), any(Client.class),
                    any(ContainerLogsRequest.class), anyString());
            cli.setConf(new YarnConfiguration());
            int exitCode = cli.run(new String[] { "-containerId", containerId1.toString() });
            assertTrue(exitCode == 0);
            assertTrue(sysOutStream.toString().contains(logMessage(containerId1, "syslog")));
            sysOutStream.reset();
        } finally {
            IOUtils.closeQuietly(fis);
            fs.delete(new Path(rootLogDir), true);
        }
    }

    @Test(timeout = 5000)
    public void testFetchRunningApplicationLogs() throws Exception {

        UserGroupInformation ugi = UserGroupInformation.getCurrentUser();

        NodeId nodeId = NodeId.newInstance("localhost", 1234);
        ApplicationId appId = ApplicationId.newInstance(0, 1);
        ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);

        // Create a mock ApplicationAttempt Report
        ApplicationAttemptReport mockAttemptReport = mock(ApplicationAttemptReport.class);
        doReturn(appAttemptId).when(mockAttemptReport).getApplicationAttemptId();
        List<ApplicationAttemptReport> attemptReports = Arrays.asList(mockAttemptReport);

        // Create two mock containerReports
        ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1);
        ContainerReport mockContainerReport1 = mock(ContainerReport.class);
        doReturn(containerId1).when(mockContainerReport1).getContainerId();
        doReturn(nodeId).when(mockContainerReport1).getAssignedNode();
        doReturn("http://localhost:2345").when(mockContainerReport1).getNodeHttpAddress();
        ContainerId containerId2 = ContainerId.newContainerId(appAttemptId, 2);
        ContainerReport mockContainerReport2 = mock(ContainerReport.class);
        doReturn(containerId2).when(mockContainerReport2).getContainerId();
        doReturn(nodeId).when(mockContainerReport2).getAssignedNode();
        doReturn("http://localhost:2345").when(mockContainerReport2).getNodeHttpAddress();
        List<ContainerReport> containerReports = Arrays.asList(mockContainerReport1, mockContainerReport2);

        // Mock the YarnClient, and it would report the previous created
        // mockAttemptReport and previous two created mockContainerReports
        YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.RUNNING, ugi.getShortUserName(), true,
                attemptReports, containerReports);
        LogsCLI cli = spy(new LogsCLIForTest(mockYarnClient));
        doReturn(0).when(cli).printContainerLogsFromRunningApplication(any(Configuration.class),
                any(ContainerLogsRequest.class), any(LogCLIHelpers.class), anyBoolean());

        cli.setConf(new YarnConfiguration());
        int exitCode = cli.run(new String[] { "-applicationId", appId.toString() });
        assertTrue(exitCode == 0);

        ArgumentCaptor<ContainerLogsRequest> logsRequestCaptor = ArgumentCaptor
                .forClass(ContainerLogsRequest.class);

        // we have two container reports, so make sure we have called
        // printContainerLogsFromRunningApplication twice
        verify(cli, times(2)).printContainerLogsFromRunningApplication(any(Configuration.class),
                logsRequestCaptor.capture(), any(LogCLIHelpers.class), anyBoolean());

        // Verify that the log-type is "ALL"
        List<ContainerLogsRequest> capturedRequests = logsRequestCaptor.getAllValues();
        Assert.assertEquals(2, capturedRequests.size());
        Set<String> logTypes0 = capturedRequests.get(0).getLogTypes();
        Set<String> logTypes1 = capturedRequests.get(1).getLogTypes();
        Assert.assertTrue(logTypes0.contains("ALL") && (logTypes0.size() == 1));
        Assert.assertTrue(logTypes1.contains("ALL") && (logTypes1.size() == 1));
    }

    @Test(timeout = 15000)
    public void testFetchApplictionLogsAsAnotherUser() throws Exception {
        String remoteLogRootDir = "target/logs/";
        String rootLogDir = "target/LocalLogs";

        String testUser = "test";
        UserGroupInformation testUgi = UserGroupInformation.createRemoteUser(testUser);

        Configuration configuration = new Configuration();
        configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
        configuration.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
        configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
        configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
        FileSystem fs = FileSystem.get(configuration);

        ApplicationId appId = ApplicationId.newInstance(0, 1);
        ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
        ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
        NodeId nodeId = NodeId.newInstance("localhost", 1234);

        try {
            Path rootLogDirPath = new Path(rootLogDir);
            if (fs.exists(rootLogDirPath)) {
                fs.delete(rootLogDirPath, true);
            }
            assertTrue(fs.mkdirs(rootLogDirPath));

            // create local app dir for app
            final Path appLogsDir = new Path(rootLogDirPath, appId.toString());
            if (fs.exists(appLogsDir)) {
                fs.delete(appLogsDir, true);
            }
            assertTrue(fs.mkdirs(appLogsDir));

            List<String> rootLogDirs = Arrays.asList(rootLogDir);
            List<String> logTypes = new ArrayList<String>();
            logTypes.add("syslog");

            // create container logs in localLogDir for app
            createContainerLogInLocalDir(appLogsDir, containerId, fs, logTypes);

            // create the remote app dir for app
            // but for a different user testUser"
            Path path = new Path(remoteLogRootDir + testUser + "/logs/" + appId);
            if (fs.exists(path)) {
                fs.delete(path, true);
            }
            assertTrue(fs.mkdirs(path));

            // upload container logs for app into remote dir
            uploadContainerLogIntoRemoteDir(testUgi, configuration, rootLogDirs, nodeId, containerId, path, fs);

            YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED,
                    testUgi.getShortUserName());
            LogsCLI cli = new LogsCLIForTest(mockYarnClient);
            cli.setConf(configuration);

            // Verify that we can get the application logs by specifying
            // a correct appOwner
            int exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-appOwner", testUser });
            assertTrue(exitCode == 0);
            assertTrue(sysOutStream.toString().contains(logMessage(containerId, "syslog")));
            sysOutStream.reset();

            // Verify that we can not get the application logs
            // if an invalid user is specified
            exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-appOwner", "invalid" });
            assertTrue(exitCode == -1);
            assertTrue(sysErrStream.toString()
                    .contains("Can not find the logs " + "for the application: " + appId.toString()));
            sysErrStream.reset();

            // Verify that we do not specify appOwner, and can not
            // get appReport from RM, we still can figure out the appOwner
            // and can get app logs successfully.
            YarnClient mockYarnClient2 = createMockYarnClientUnknownApp();
            cli = new LogsCLIForTest(mockYarnClient2);
            cli.setConf(configuration);
            exitCode = cli.run(new String[] { "-applicationId", appId.toString() });
            assertTrue(exitCode == 0);
            assertTrue(sysOutStream.toString().contains(logMessage(containerId, "syslog")));
            sysOutStream.reset();

            // Verify that we could get the err message "Can not find the appOwner"
            // if we do not specify the appOwner, can not get appReport, and
            // the app does not exist in remote dir.
            ApplicationId appId2 = ApplicationId.newInstance(System.currentTimeMillis(), 2);
            exitCode = cli.run(new String[] { "-applicationId", appId2.toString() });
            assertTrue(exitCode == -1);
            assertTrue(sysErrStream.toString().contains("Can not find the appOwner"));
            sysErrStream.reset();

            // Verify that we could not get appOwner
            // because we don't have file-system permissions
            ApplicationId appTest = ApplicationId.newInstance(System.currentTimeMillis(), 1000);
            String priorityUser = "priority";
            Path pathWithoutPerm = new Path(remoteLogRootDir + priorityUser + "/logs/" + appTest);
            if (fs.exists(pathWithoutPerm)) {
                fs.delete(pathWithoutPerm, true);
            }
            // The user will not have read permission for this directory.
            // To mimic the scenario that the user can not get file status
            FsPermission permission = FsPermission.createImmutable((short) 01300);
            assertTrue(fs.mkdirs(pathWithoutPerm, permission));

            exitCode = cli.run(new String[] { "-applicationId", appTest.toString() });
            assertTrue(exitCode == -1);
            assertTrue(sysErrStream.toString()
                    .contains("Can not find the logs for the application: " + appTest.toString()));
            sysErrStream.reset();
        } finally {
            fs.delete(new Path(remoteLogRootDir), true);
            fs.delete(new Path(rootLogDir), true);
        }
    }

    @Test(timeout = 5000)
    public void testLogsCLIWithInvalidArgs() throws Exception {
        String localDir = "target/SaveLogs";
        Path localPath = new Path(localDir);
        Configuration configuration = new Configuration();
        FileSystem fs = FileSystem.get(configuration);
        ApplicationId appId = ApplicationId.newInstance(0, 1);
        YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED,
                UserGroupInformation.getCurrentUser().getShortUserName());
        LogsCLI cli = new LogsCLIForTest(mockYarnClient);
        cli.setConf(configuration);

        // Specify an invalid applicationId
        int exitCode = cli.run(new String[] { "-applicationId", "123" });
        assertTrue(exitCode == -1);
        assertTrue(sysErrStream.toString().contains("Invalid ApplicationId specified"));
        sysErrStream.reset();

        // Specify an invalid containerId
        exitCode = cli.run(new String[] { "-containerId", "123" });
        assertTrue(exitCode == -1);
        assertTrue(sysErrStream.toString().contains("Invalid ContainerId specified"));
        sysErrStream.reset();

        // Specify show_container_log_info and show_application_log_info
        // at the same time
        exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-show_container_log_info",
                "-show_application_log_info" });
        assertTrue(exitCode == -1);
        assertTrue(sysErrStream.toString().contains("Invalid options. "
                + "Can only accept one of show_application_log_info/" + "show_container_log_info."));
        sysErrStream.reset();

        // Specify log_files and log_files_pattern
        // at the same time
        exitCode = cli.run(
                new String[] { "-applicationId", appId.toString(), "-log_files", "*", "-log_files_pattern", ".*" });
        assertTrue(exitCode == -1);
        assertTrue(sysErrStream.toString()
                .contains("Invalid options. " + "Can only accept one of log_files/" + "log_files_pattern."));
        sysErrStream.reset();

        // Specify a file name to the option -out
        try {
            fs.mkdirs(localPath);
            Path tmpFilePath = new Path(localPath, "tmpFile");
            if (!fs.exists(tmpFilePath)) {
                fs.createNewFile(tmpFilePath);
            }
            exitCode = cli.run(new String[] { "-applicationId", appId.toString(), "-out", tmpFilePath.toString() });
            assertTrue(exitCode == -1);
            assertTrue(
                    sysErrStream.toString().contains("Invalid value for -out option. Please provide a directory."));
        } finally {
            fs.delete(localPath, true);
        }
    }

    @Test(timeout = 15000)
    public void testSaveContainerLogsLocally() throws Exception {
        String remoteLogRootDir = "target/logs/";
        String rootLogDir = "target/LocalLogs";
        String localDir = "target/SaveLogs";
        Path localPath = new Path(localDir);

        Configuration configuration = new Configuration();
        configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
        configuration.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
        configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
        configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");

        FileSystem fs = FileSystem.get(configuration);
        ApplicationId appId = ApplicationId.newInstance(0, 1);
        ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
        List<ContainerId> containerIds = new ArrayList<ContainerId>();
        ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1);
        ContainerId containerId2 = ContainerId.newContainerId(appAttemptId, 2);
        containerIds.add(containerId1);
        containerIds.add(containerId2);

        List<NodeId> nodeIds = new ArrayList<NodeId>();
        NodeId nodeId = NodeId.newInstance("localhost", 1234);
        NodeId nodeId2 = NodeId.newInstance("test", 4567);
        nodeIds.add(nodeId);
        nodeIds.add(nodeId2);

        try {
            createContainerLogs(configuration, remoteLogRootDir, rootLogDir, fs, appId, containerIds, nodeIds);

            YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED,
                    UserGroupInformation.getCurrentUser().getShortUserName());
            LogsCLI cli = new LogsCLIForTest(mockYarnClient);
            cli.setConf(configuration);
            int exitCode = cli
                    .run(new String[] { "-applicationId", appId.toString(), "-out", localPath.toString() });
            assertTrue(exitCode == 0);

            // make sure we created a dir named as node id
            FileStatus[] nodeDir = fs.listStatus(localPath);
            Arrays.sort(nodeDir);
            assertTrue(nodeDir.length == 2);
            assertTrue(nodeDir[0].getPath().getName().contains(LogAggregationUtils.getNodeString(nodeId)));
            assertTrue(nodeDir[1].getPath().getName().contains(LogAggregationUtils.getNodeString(nodeId2)));

            FileStatus[] container1Dir = fs.listStatus(nodeDir[0].getPath());
            assertTrue(container1Dir.length == 1);
            assertTrue(container1Dir[0].getPath().getName().equals(containerId1.toString()));
            String container1 = readContainerContent(container1Dir[0].getPath(), fs);
            assertTrue(container1.contains(logMessage(containerId1, "syslog")));

            FileStatus[] container2Dir = fs.listStatus(nodeDir[1].getPath());
            assertTrue(container2Dir.length == 1);
            assertTrue(container2Dir[0].getPath().getName().equals(containerId2.toString()));
            String container2 = readContainerContent(container2Dir[0].getPath(), fs);
            assertTrue(container2.contains(logMessage(containerId2, "syslog")));
        } finally {
            fs.delete(new Path(remoteLogRootDir), true);
            fs.delete(new Path(rootLogDir), true);
            fs.delete(localPath, true);
        }
    }

    private String readContainerContent(Path containerPath, FileSystem fs) throws IOException {
        assertTrue(fs.exists(containerPath));
        StringBuffer inputLine = new StringBuffer();
        BufferedReader reader = null;
        try {
            reader = new BufferedReader(new InputStreamReader(fs.open(containerPath)));
            String tmp;
            while ((tmp = reader.readLine()) != null) {
                inputLine.append(tmp);
            }
            return inputLine.toString();
        } finally {
            if (reader != null) {
                IOUtils.closeQuietly(reader);
            }
        }
    }

    @Test(timeout = 15000)
    public void testPrintContainerLogMetadata() throws Exception {
        String remoteLogRootDir = "target/logs/";
        Configuration configuration = new Configuration();
        configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
        configuration.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
        configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
        configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
        FileSystem fs = FileSystem.get(configuration);
        String rootLogDir = "target/LocalLogs";

        ApplicationId appId = ApplicationId.newInstance(0, 1);
        ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
        List<ContainerId> containerIds = new ArrayList<ContainerId>();
        ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1);
        ContainerId containerId2 = ContainerId.newContainerId(appAttemptId, 2);
        containerIds.add(containerId1);
        containerIds.add(containerId2);

        List<NodeId> nodeIds = new ArrayList<NodeId>();
        NodeId nodeId = NodeId.newInstance("localhost", 1234);
        nodeIds.add(nodeId);
        nodeIds.add(nodeId);

        createContainerLogs(configuration, remoteLogRootDir, rootLogDir, fs, appId, containerIds, nodeIds);

        YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED,
                UserGroupInformation.getCurrentUser().getShortUserName());
        LogsCLI cli = new LogsCLIForTest(mockYarnClient);
        cli.setConf(configuration);

        cli.run(new String[] { "-applicationId", appId.toString(), "-show_container_log_info" });
        assertTrue(sysOutStream.toString().contains("Container: container_0_0001_01_000001 on localhost_"));
        assertTrue(sysOutStream.toString().contains("Container: container_0_0001_01_000002 on localhost_"));
        assertTrue(sysOutStream.toString().contains("syslog"));
        assertTrue(sysOutStream.toString().contains("43"));
        sysOutStream.reset();

        cli.run(new String[] { "-applicationId", appId.toString(), "-show_container_log_info", "-containerId",
                "container_0_0001_01_000001" });
        assertTrue(sysOutStream.toString().contains("Container: container_0_0001_01_000001 on localhost_"));
        assertFalse(sysOutStream.toString().contains("Container: container_0_0001_01_000002 on localhost_"));
        assertTrue(sysOutStream.toString().contains("syslog"));
        assertTrue(sysOutStream.toString().contains("43"));
        sysOutStream.reset();

        cli.run(new String[] { "-applicationId", appId.toString(), "-show_container_log_info", "-nodeAddress",
                "localhost" });
        assertTrue(sysOutStream.toString().contains("Container: container_0_0001_01_000001 on localhost_"));
        assertTrue(sysOutStream.toString().contains("Container: container_0_0001_01_000002 on localhost_"));
        assertTrue(sysOutStream.toString().contains("syslog"));
        assertTrue(sysOutStream.toString().contains("43"));
        sysOutStream.reset();

        cli.run(new String[] { "-applicationId", appId.toString(), "-show_container_log_info", "-nodeAddress",
                "localhost", "-containerId", "container_1234" });
        assertTrue(sysErrStream.toString().contains("Invalid ContainerId specified"));
        sysErrStream.reset();

        cli.run(new String[] { "-applicationId", appId.toString(), "-show_application_log_info" });
        assertTrue(sysOutStream.toString().contains("Application State: Completed."));
        assertTrue(sysOutStream.toString().contains("container_0_0001_01_000001 on localhost"));
        assertTrue(sysOutStream.toString().contains("container_0_0001_01_000002 on localhost"));
        sysOutStream.reset();

        cli.run(new String[] { "-applicationId", appId.toString(), "-show_application_log_info", "-nodeAddress",
                "localhost" });
        assertTrue(sysOutStream.toString().contains("Application State: Completed."));
        assertTrue(sysOutStream.toString().contains("container_0_0001_01_000001 on localhost"));
        assertTrue(sysOutStream.toString().contains("container_0_0001_01_000002 on localhost"));
        sysOutStream.reset();

        fs.delete(new Path(remoteLogRootDir), true);
        fs.delete(new Path(rootLogDir), true);
    }

    @Test(timeout = 15000)
    public void testListNodeInfo() throws Exception {
        String remoteLogRootDir = "target/logs/";
        Configuration configuration = new Configuration();
        configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
        configuration.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
        configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
        configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");

        ApplicationId appId = ApplicationId.newInstance(0, 1);
        ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
        List<ContainerId> containerIds = new ArrayList<ContainerId>();
        ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1);
        ContainerId containerId2 = ContainerId.newContainerId(appAttemptId, 2);
        containerIds.add(containerId1);
        containerIds.add(containerId2);

        List<NodeId> nodeIds = new ArrayList<NodeId>();
        NodeId nodeId1 = NodeId.newInstance("localhost1", 1234);
        NodeId nodeId2 = NodeId.newInstance("localhost2", 2345);
        nodeIds.add(nodeId1);
        nodeIds.add(nodeId2);

        String rootLogDir = "target/LocalLogs";
        FileSystem fs = FileSystem.get(configuration);

        createContainerLogs(configuration, remoteLogRootDir, rootLogDir, fs, appId, containerIds, nodeIds);

        YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED,
                UserGroupInformation.getCurrentUser().getShortUserName());
        LogsCLI cli = new LogsCLIForTest(mockYarnClient);
        cli.setConf(configuration);

        cli.run(new String[] { "-applicationId", appId.toString(), "-list_nodes" });
        assertTrue(sysOutStream.toString().contains(LogAggregationUtils.getNodeString(nodeId1)));
        assertTrue(sysOutStream.toString().contains(LogAggregationUtils.getNodeString(nodeId2)));
        sysOutStream.reset();

        fs.delete(new Path(remoteLogRootDir), true);
        fs.delete(new Path(rootLogDir), true);
    }

    private void createContainerLogs(Configuration configuration, String remoteLogRootDir, String rootLogDir,
            FileSystem fs, ApplicationId appId, List<ContainerId> containerIds, List<NodeId> nodeIds)
            throws Exception {

        UserGroupInformation ugi = UserGroupInformation.getCurrentUser();

        // create local logs
        Path rootLogDirPath = new Path(rootLogDir);
        if (fs.exists(rootLogDirPath)) {
            fs.delete(rootLogDirPath, true);
        }
        assertTrue(fs.mkdirs(rootLogDirPath));

        Path appLogsDir = new Path(rootLogDirPath, appId.toString());
        if (fs.exists(appLogsDir)) {
            fs.delete(appLogsDir, true);
        }
        assertTrue(fs.mkdirs(appLogsDir));
        List<String> rootLogDirs = Arrays.asList(rootLogDir);
        List<String> logTypes = new ArrayList<String>();
        logTypes.add("syslog");
        // create container logs in localLogDir
        for (ContainerId containerId : containerIds) {
            createContainerLogInLocalDir(appLogsDir, containerId, fs, logTypes);
        }
        Path path = new Path(remoteLogRootDir + ugi.getShortUserName() + "/logs/application_0_0001");

        if (fs.exists(path)) {
            fs.delete(path, true);
        }
        assertTrue(fs.mkdirs(path));
        for (int i = 0; i < containerIds.size(); i++) {
            uploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeIds.get(i), containerIds.get(i),
                    path, fs);
        }
    }

    private static void createContainerLogInLocalDir(Path appLogsDir, ContainerId containerId, FileSystem fs,
            List<String> logTypes) throws Exception {
        Path containerLogsDir = new Path(appLogsDir, containerId.toString());
        if (fs.exists(containerLogsDir)) {
            fs.delete(containerLogsDir, true);
        }
        assertTrue(fs.mkdirs(containerLogsDir));
        for (String logType : logTypes) {
            Writer writer = new FileWriter(new File(containerLogsDir.toString(), logType));
            writer.write(logMessage(containerId, logType));
            writer.close();
        }
    }

    private static String logMessage(ContainerId containerId, String logType) {
        StringBuilder sb = new StringBuilder();
        sb.append("Hello " + containerId + " in " + logType + "!");
        return sb.toString();
    }

    private static void uploadContainerLogIntoRemoteDir(UserGroupInformation ugi, Configuration configuration,
            List<String> rootLogDirs, NodeId nodeId, ContainerId containerId, Path appDir, FileSystem fs)
            throws Exception {
        Path path = new Path(appDir, LogAggregationUtils.getNodeString(nodeId) + System.currentTimeMillis());
        AggregatedLogFormat.LogWriter writer = new AggregatedLogFormat.LogWriter(configuration, path, ugi);
        writer.writeApplicationOwner(ugi.getUserName());

        Map<ApplicationAccessType, String> appAcls = new HashMap<ApplicationAccessType, String>();
        appAcls.put(ApplicationAccessType.VIEW_APP, ugi.getUserName());
        writer.writeApplicationACLs(appAcls);
        writer.append(new AggregatedLogFormat.LogKey(containerId), new AggregatedLogFormat.LogValue(rootLogDirs,
                containerId, UserGroupInformation.getCurrentUser().getShortUserName()));
        writer.close();
    }

    private static void uploadEmptyContainerLogIntoRemoteDir(UserGroupInformation ugi, Configuration configuration,
            List<String> rootLogDirs, NodeId nodeId, ContainerId containerId, Path appDir, FileSystem fs)
            throws Exception {
        Path path = new Path(appDir, LogAggregationUtils.getNodeString(nodeId) + System.currentTimeMillis());
        AggregatedLogFormat.LogWriter writer = new AggregatedLogFormat.LogWriter(configuration, path, ugi);
        writer.writeApplicationOwner(ugi.getUserName());

        Map<ApplicationAccessType, String> appAcls = new HashMap<ApplicationAccessType, String>();
        appAcls.put(ApplicationAccessType.VIEW_APP, ugi.getUserName());
        writer.writeApplicationACLs(appAcls);
        DataOutputStream out = writer.getWriter().prepareAppendKey(-1);
        new AggregatedLogFormat.LogKey(containerId).write(out);
        out.close();
        out = writer.getWriter().prepareAppendValue(-1);
        new AggregatedLogFormat.LogValue(rootLogDirs, containerId,
                UserGroupInformation.getCurrentUser().getShortUserName()).write(out, new HashSet<File>());
        out.close();
        writer.close();
    }

    private YarnClient createMockYarnClient(YarnApplicationState appState, String user)
            throws YarnException, IOException {
        return createMockYarnClient(appState, user, false, null, null);
    }

    private YarnClient createMockYarnClient(YarnApplicationState appState, String user, boolean mockContainerReport,
            List<ApplicationAttemptReport> mockAttempts, List<ContainerReport> mockContainers)
            throws YarnException, IOException {
        YarnClient mockClient = mock(YarnClient.class);
        ApplicationReport mockAppReport = mock(ApplicationReport.class);
        doReturn(user).when(mockAppReport).getUser();
        doReturn(appState).when(mockAppReport).getYarnApplicationState();
        doReturn(mockAppReport).when(mockClient).getApplicationReport(any(ApplicationId.class));
        if (mockContainerReport) {
            doReturn(mockAttempts).when(mockClient).getApplicationAttempts(any(ApplicationId.class));
            doReturn(mockContainers).when(mockClient).getContainers(any(ApplicationAttemptId.class));
        }
        return mockClient;
    }

    private YarnClient createMockYarnClientWithException() throws YarnException, IOException {
        YarnClient mockClient = mock(YarnClient.class);
        doThrow(new YarnException()).when(mockClient).getApplicationReport(any(ApplicationId.class));
        doThrow(new YarnException()).when(mockClient).getContainerReport(any(ContainerId.class));
        return mockClient;
    }

    private YarnClient createMockYarnClientUnknownApp() throws YarnException, IOException {
        YarnClient mockClient = mock(YarnClient.class);
        doThrow(new YarnException("Unknown AppId")).when(mockClient).getApplicationReport(any(ApplicationId.class));
        return mockClient;
    }

    private static class LogsCLIForTest extends LogsCLI {

        private YarnClient yarnClient;

        public LogsCLIForTest(YarnClient yarnClient) {
            super();
            this.yarnClient = yarnClient;
        }

        protected YarnClient createYarnClient() {
            return yarnClient;
        }
    }
}