com.netflix.genie.client.JobClientIntegrationTests.java Source code

Java tutorial

Introduction

Here is the source code for com.netflix.genie.client.JobClientIntegrationTests.java

Source

/*
 *
 *  Copyright 2015 Netflix, Inc.
 *
 *     Licensed under the Apache License, Version 2.0 (the "License");
 *     you may not use this file except in compliance with the License.
 *     You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *     Unless required by applicable law or agreed to in writing, software
 *     distributed under the License is distributed on an "AS IS" BASIS,
 *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *     See the License for the specific language governing permissions and
 *     limitations under the License.
 *
 */
package com.netflix.genie.client;

import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.netflix.genie.client.configs.GenieNetworkConfiguration;
import com.netflix.genie.client.exceptions.GenieClientException;
import com.netflix.genie.common.dto.Cluster;
import com.netflix.genie.common.dto.ClusterCriteria;
import com.netflix.genie.common.dto.ClusterStatus;
import com.netflix.genie.common.dto.Command;
import com.netflix.genie.common.dto.CommandStatus;
import com.netflix.genie.common.dto.Job;
import com.netflix.genie.common.dto.JobExecution;
import com.netflix.genie.common.dto.JobRequest;
import com.netflix.genie.common.dto.JobStatus;
import com.netflix.genie.common.dto.JobStatusMessages;
import com.netflix.genie.common.dto.search.JobSearchResult;
import lombok.extern.slf4j.Slf4j;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.springframework.core.io.DefaultResourceLoader;
import org.springframework.core.io.ResourceLoader;

import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

/**
 * Integration tests for Genie Job Client.
 *
 * @author amsharma
 * @since 3.0.0
 */
@Slf4j
@Ignore
public class JobClientIntegrationTests extends GenieClientsIntegrationTestsBase {

    private static final String JOB_NAME = "List Directories bash job";
    private static final String JOB_USER = "genie";
    private static final String JOB_VERSION = "1.0";
    private static final String JOB_DESCRIPTION = "Genie 3 Test Job";
    private static final String CLUSTER_NAME = "Cluster Name";
    private static final String COMMAND_NAME = "Command Name";

    private ClusterClient clusterClient;
    private CommandClient commandClient;
    //private ApplicationClient applicationClient;
    private JobClient jobClient;
    private ResourceLoader resourceLoader;

    /**
     * Setup for tests.
     *
     * @throws Exception If there is an error.
     */
    @Before
    public void setup() throws Exception {
        this.resourceLoader = new DefaultResourceLoader();
        clusterClient = new ClusterClient(getBaseUrl(), null, null);
        commandClient = new CommandClient(getBaseUrl(), null, null);
        //applicationClient = new ApplicationClient(getBaseUrl());
        final GenieNetworkConfiguration genieNetworkConfiguration = new GenieNetworkConfiguration();
        genieNetworkConfiguration.setReadTimeout(20000);
        jobClient = new JobClient(getBaseUrl(), null, genieNetworkConfiguration);
    }

    /**
     * Method to test submitting a job.
     *
     * @throws Exception If there is any problem.
     */
    @Test
    public void canSubmitJob() throws Exception {

        createClusterAndCommandForTest();

        final String jobId = UUID.randomUUID().toString();

        final List<ClusterCriteria> clusterCriteriaList = Lists
                .newArrayList(new ClusterCriteria(Sets.newHashSet("laptop")));

        final Set<String> commandCriteria = Sets.newHashSet("bash");

        final String configFile1 = this.resourceLoader.getResource("/config1").getFile().getAbsolutePath();
        final Set<String> configs = Sets.newHashSet(configFile1);

        final String depFile1 = this.resourceLoader.getResource("/dep1").getFile().getAbsolutePath();
        final Set<String> dependencies = Sets.newHashSet(depFile1);

        final String setUpFile = this.resourceLoader.getResource("/setupfile").getFile().getAbsolutePath();

        final JobRequest jobRequest = new JobRequest.Builder(JOB_NAME, JOB_USER, JOB_VERSION,
                "-c 'echo HELLO WORLD!!!'", clusterCriteriaList, commandCriteria).withId(jobId)
                        .withDisableLogArchival(true).withSetupFile(setUpFile).withConfigs(configs)
                        .withDependencies(dependencies).withDescription(JOB_DESCRIPTION).build();

        final String id = jobClient.submitJob(jobRequest);

        final JobStatus jobStatus = jobClient.waitForCompletion(jobId, 600000);

        Assert.assertEquals(JobStatus.SUCCEEDED, jobStatus);
        final Job job = jobClient.getJob(id);

        Assert.assertEquals(jobId, job.getId().orElseThrow(IllegalArgumentException::new));

        final JobRequest jobRequest1 = jobClient.getJobRequest(jobId);
        Assert.assertEquals(jobId, jobRequest1.getId().orElseThrow(IllegalArgumentException::new));
        Assert.assertTrue(jobRequest1.getDependencies().contains(depFile1));

        final JobExecution jobExecution = jobClient.getJobExecution(jobId);
        Assert.assertEquals(jobId, jobExecution.getId().orElseThrow(IllegalArgumentException::new));

        Assert.assertEquals(CLUSTER_NAME, jobClient.getJobCluster(jobId).getName());
        Assert.assertEquals(COMMAND_NAME, jobClient.getJobCommand(jobId).getName());
    }

    /**
     * Method to test submitting/killing a job.
     *
     * @throws Exception If there is any problem.
     */
    @Test
    public void submitAndKillJob() throws Exception {

        createClusterAndCommandForTest();

        final String jobId = UUID.randomUUID().toString();

        final List<ClusterCriteria> clusterCriteriaList = Lists
                .newArrayList(new ClusterCriteria(Sets.newHashSet("laptop")));

        final Set<String> commandCriteria = Sets.newHashSet("bash");

        final String depFile1 = this.resourceLoader.getResource("/dep1").getFile().getAbsolutePath();
        final Set<String> dependencies = Sets.newHashSet(depFile1);

        final String setUpFile = this.resourceLoader.getResource("/setupfile").getFile().getAbsolutePath();

        final JobRequest jobRequest = new JobRequest.Builder(JOB_NAME, JOB_USER, JOB_VERSION,
                "-c 'echo HELLO WORLD!!!'", clusterCriteriaList, commandCriteria).withId(jobId)
                        .withDisableLogArchival(true).withSetupFile(setUpFile).withDependencies(dependencies)
                        .withDescription(JOB_DESCRIPTION).build();

        final ExecutorService executors = Executors.newFixedThreadPool(2);
        final Future<String> jobFuture;
        try {
            jobFuture = executors.submit(() -> jobClient.submitJob(jobRequest));
            executors.submit(() -> {
                boolean result = true;
                while (result) {
                    try {
                        jobClient.getJob(jobId);
                        jobClient.killJob(jobId);
                        Thread.sleep(1000);
                        result = false;
                    } catch (Exception ignored) {
                        result = true;
                    }
                }
            });
        } finally {
            executors.shutdown();
            executors.awaitTermination(Integer.MAX_VALUE, TimeUnit.HOURS);
        }
        final Job job = jobClient.getJob(jobId);
        Assert.assertEquals(jobId, jobFuture.get());
        Assert.assertEquals(JobStatus.KILLED, job.getStatus());
    }

    /**
     * Method to test submitting a job using attachments.
     *
     * @throws Exception If there is any problem.
     */
    @Test
    public void testJobSubmissionWithAttachmentsFiles() throws Exception {

        try (final InputStream att1 = new FileInputStream(
                this.resourceLoader.getResource("/setupfile").getFile().getAbsolutePath());
                final InputStream att2 = new FileInputStream(
                        this.resourceLoader.getResource("/data.txt").getFile().getAbsolutePath())) {

            createClusterAndCommandForTest();
            final String jobId = UUID.randomUUID().toString();

            final List<ClusterCriteria> clusterCriteriaList = Lists
                    .newArrayList(new ClusterCriteria(Sets.newHashSet("laptop")));

            final Set<String> commandCriteria = Sets.newHashSet("bash");

            final JobRequest jobRequest = new JobRequest.Builder(JOB_NAME, JOB_USER, JOB_VERSION,
                    "-c 'cat data.txt'", clusterCriteriaList, commandCriteria).withId(jobId)
                            .withDisableLogArchival(true).build();

            final Map<String, InputStream> attachments = new HashMap<>();
            attachments.put("setupfile", att1);
            attachments.put("data.txt", att2);
            final String id = jobClient.submitJobWithAttachments(jobRequest, attachments);

            final JobStatus jobStatus = jobClient.waitForCompletion(jobId, 600000, 5000);

            Assert.assertEquals(JobStatus.SUCCEEDED, jobStatus);
            final Job job = jobClient.getJob(id);

            Assert.assertEquals(jobId, job.getId().orElseThrow(IllegalArgumentException::new));

            final JobRequest jobRequest1 = jobClient.getJobRequest(jobId);
            Assert.assertEquals(jobId, jobRequest1.getId().orElseThrow(IllegalArgumentException::new));
        }
    }

    /**
     * Method to test submitting a job using attachments.
     *
     * @throws Exception If there is any problem.
     */
    @Test
    public void testJobSubmissionWithAttachmentsByteArray() throws Exception {

        createClusterAndCommandForTest();
        final String jobId = UUID.randomUUID().toString();

        final List<ClusterCriteria> clusterCriteriaList = Lists
                .newArrayList(new ClusterCriteria(Sets.newHashSet("laptop")));

        final Set<String> commandCriteria = Sets.newHashSet("bash");

        final JobRequest jobRequest = new JobRequest.Builder(JOB_NAME, JOB_USER, JOB_VERSION,
                "-c 'cat attachmentfile.txt'", clusterCriteriaList, commandCriteria).withId(jobId)
                        .withDisableLogArchival(true).build();

        try (final ByteArrayInputStream bis = new ByteArrayInputStream("ATTACHMENT DATA".getBytes("UTF-8"))) {
            final Map<String, InputStream> attachments = new HashMap<>();

            attachments.put("attachmentfile.txt", bis);
            jobClient.submitJobWithAttachments(jobRequest, attachments);
        }

        final JobStatus jobStatus = jobClient.waitForCompletion(jobId, 600000, 5000);
        Assert.assertEquals(JobStatus.SUCCEEDED, jobStatus);
        final Job job = jobClient.getJob(jobId);

        Assert.assertEquals(jobId, job.getId().orElseThrow(IllegalArgumentException::new));

        final InputStream inputStream1 = jobClient
                .getJobStdout(jobRequest.getId().orElseThrow(IllegalArgumentException::new));
        final BufferedReader reader1 = new BufferedReader(new InputStreamReader(inputStream1, "UTF-8"));
        final StringBuilder sb = new StringBuilder();
        String line;
        while ((line = reader1.readLine()) != null) {
            sb.append(line);
        }

        reader1.close();
        inputStream1.close();

        Assert.assertEquals("ATTACHMENT DATA", sb.toString());
    }

    /**
     * Method to test killing a job.
     *
     * @throws Exception If there is any problem.
     */
    @Test
    public void testJobKill() throws Exception {
        createClusterAndCommandForTest();

        final List<ClusterCriteria> clusterCriteriaList = Lists
                .newArrayList(new ClusterCriteria(Sets.newHashSet("laptop")));

        final Set<String> commandCriteria = Sets.newHashSet("bash");

        final JobRequest jobRequest1 = new JobRequest.Builder(JOB_NAME, JOB_USER, JOB_VERSION, "-c 'sleep 60'",
                clusterCriteriaList, commandCriteria).withId(UUID.randomUUID().toString())
                        .withDisableLogArchival(true).build();

        jobClient.submitJob(jobRequest1);
        Thread.sleep(2000);
        jobClient.killJob(jobRequest1.getId().orElseThrow(IllegalArgumentException::new));

        final JobStatus jobStatus = jobClient
                .waitForCompletion(jobRequest1.getId().orElseThrow(IllegalArgumentException::new), 600000, 5000);

        Assert.assertEquals(JobStatus.KILLED, jobStatus);
        Assert.assertEquals(JobStatusMessages.JOB_KILLED_BY_USER,
                jobClient.getJob(jobRequest1.getId().orElseThrow(IllegalArgumentException::new)).getStatusMsg()
                        .orElseThrow(IllegalArgumentException::new));

    }

    /**
     * Method to test job getting killed by JobMonitor due to timeout.
     *
     * @throws Exception If there is any problem.
     */
    @Test
    public void testJobKillDueToTimeout() throws Exception {
        createClusterAndCommandForTest();

        final List<ClusterCriteria> clusterCriteriaList = Lists
                .newArrayList(new ClusterCriteria(Sets.newHashSet("laptop")));

        final Set<String> commandCriteria = Sets.newHashSet("bash");

        final JobRequest jobRequest1 = new JobRequest.Builder(JOB_NAME, JOB_USER, JOB_VERSION, "-c 'sleep 60'",
                clusterCriteriaList, commandCriteria).withId(UUID.randomUUID().toString())
                        .withDisableLogArchival(true).withTimeout(1).build();

        jobClient.submitJob(jobRequest1);
        Thread.sleep(2000);

        final JobStatus jobStatus = jobClient
                .waitForCompletion(jobRequest1.getId().orElseThrow(IllegalArgumentException::new), 600000, 5000);

        Assert.assertEquals(JobStatus.KILLED, jobStatus);
        Assert.assertEquals("Job exceeded timeout.",
                jobClient.getJob(jobRequest1.getId().orElseThrow(IllegalArgumentException::new)).getStatusMsg()
                        .orElseThrow(IllegalArgumentException::new));
    }

    /**
     * Method to test getJobs function.
     *
     * @throws Exception If there is a problem.
     */
    @Test
    public void testCanGetJobs() throws Exception {

        createClusterAndCommandForTest();

        final List<ClusterCriteria> clusterCriteriaList = Lists
                .newArrayList(new ClusterCriteria(Sets.newHashSet("laptop")));

        final Set<String> commandCriteria = Sets.newHashSet("bash");

        final JobRequest jobRequest1 = new JobRequest.Builder(JOB_NAME, JOB_USER, JOB_VERSION,
                "-c 'echo HELLO WORLD!!!'", clusterCriteriaList, commandCriteria)
                        .withId(UUID.randomUUID().toString()).withDisableLogArchival(true).build();

        jobClient.submitJob(jobRequest1);

        final JobRequest jobRequest2 = new JobRequest.Builder(JOB_NAME, JOB_USER, JOB_VERSION,
                "-c 'echo HELLO WORLD!!!'", clusterCriteriaList, commandCriteria)
                        .withId(UUID.randomUUID().toString()).withDisableLogArchival(true).build();

        jobClient.submitJob(jobRequest2);

        final List<JobSearchResult> jobs = jobClient.getJobs();
        Assert.assertTrue(jobs.size() >= 2);
    }

    /**
     * Method to test getJobs with params function.
     *
     * @throws Exception If there is a problem.
     */
    @Test
    public void testCanGetJobsUsingParams() throws Exception {

        createClusterAndCommandForTest();

        final List<ClusterCriteria> clusterCriteriaList = Lists
                .newArrayList(new ClusterCriteria(Sets.newHashSet("laptop")));

        final Set<String> commandCriteria = Sets.newHashSet("bash");

        final JobRequest jobRequest1 = new JobRequest.Builder("job1", "user1", JOB_VERSION,
                "-c 'echo HELLO WORLD!!!'", clusterCriteriaList, commandCriteria)
                        .withTags(Arrays.stream(new String[] { "foo", "bar" }).collect(Collectors.toSet()))
                        .withId(UUID.randomUUID().toString()).withDisableLogArchival(true).build();

        jobClient.submitJob(jobRequest1);

        final JobRequest jobRequest2 = new JobRequest.Builder("job2", "user2", JOB_VERSION, "-c 'ls blah'",
                clusterCriteriaList, commandCriteria)
                        .withTags(Arrays.stream(new String[] { "foo", "pi" }).collect(Collectors.toSet()))
                        .withId(UUID.randomUUID().toString()).withDisableLogArchival(true).build();

        jobClient.submitJob(jobRequest2);

        jobClient.waitForCompletion(jobRequest1.getId().orElseThrow(IllegalArgumentException::new), 60000, 5000);
        jobClient.waitForCompletion(jobRequest2.getId().orElseThrow(IllegalArgumentException::new), 60000, 5000);

        // Get jobs using id
        Assert.assertEquals(jobRequest1.getId().orElseThrow(IllegalArgumentException::new),
                jobClient.getJobs(jobRequest1.getId().orElseThrow(IllegalArgumentException::new), null, null, null,
                        null, null, null, null, null, null, null, null, null).get(0).getId());

        // Get jobs using user
        Assert.assertEquals(jobRequest1.getId().orElseThrow(IllegalArgumentException::new),
                jobClient.getJobs(null, "job1", null, null, null, null, null, null, null, null, null, null, null)
                        .get(0).getId());

        // Get jobs using name
        Assert.assertEquals(jobRequest2.getId().orElseThrow(IllegalArgumentException::new),
                jobClient.getJobs(null, null, "user2", null, null, null, null, null, null, null, null, null, null)
                        .get(0).getId());

        // Get jobs using status
        Assert.assertEquals(jobRequest1.getId().orElseThrow(IllegalArgumentException::new),
                jobClient.getJobs(null, null, null,
                        Arrays.stream(new String[] { "SUCCEEDED" }).collect(Collectors.toSet()), null, null, null,
                        null, null, null, null, null, null).get(0).getId());

        // Get jobs using status
        Assert.assertEquals(jobRequest2.getId().orElseThrow(IllegalArgumentException::new),
                jobClient.getJobs(null, null, null,
                        Arrays.stream(new String[] { "FAILED" }).collect(Collectors.toSet()), null, null, null,
                        null, null, null, null, null, null).get(0).getId());

        // Get jobs using tags
        Assert.assertEquals(2,
                jobClient.getJobs(null, null, null, null,
                        Arrays.stream(new String[] { "foo" }).collect(Collectors.toSet()), null, null, null, null,
                        null, null, null, null).size());
    }

    /**
     * Method to test get stdout function.
     *
     * @throws Exception If there is a problem.
     */
    @Test
    public void testCanGetJobStdout() throws Exception {

        createClusterAndCommandForTest();

        final List<ClusterCriteria> clusterCriteriaList = Lists
                .newArrayList(new ClusterCriteria(Sets.newHashSet("laptop")));

        final Set<String> commandCriteria = Sets.newHashSet("bash");

        final JobRequest jobRequest1 = new JobRequest.Builder(JOB_NAME, JOB_USER, JOB_VERSION,
                "-c 'echo HELLO WORLD!!!'", clusterCriteriaList, commandCriteria)
                        .withId(UUID.randomUUID().toString()).withDisableLogArchival(true).build();

        jobClient.submitJob(jobRequest1);
        jobClient.waitForCompletion(jobRequest1.getId().orElseThrow(IllegalArgumentException::new), 60000, 5000);

        final InputStream inputStream1 = jobClient
                .getJobStdout(jobRequest1.getId().orElseThrow(IllegalArgumentException::new));
        final BufferedReader reader1 = new BufferedReader(new InputStreamReader(inputStream1, "UTF-8"));
        final StringBuilder sb = new StringBuilder();
        String line;
        while ((line = reader1.readLine()) != null) {
            sb.append(line);
        }

        reader1.close();
        inputStream1.close();

        Assert.assertEquals("HELLO WORLD!!!", sb.toString());
    }

    /**
     * Method to test get stdout function.
     *
     * @throws Exception If there is a problem.
     */
    @Test
    public void testCanGetJobStderr() throws Exception {

        createClusterAndCommandForTest();

        final List<ClusterCriteria> clusterCriteriaList = Lists
                .newArrayList(new ClusterCriteria(Sets.newHashSet("laptop")));

        final Set<String> commandCriteria = Sets.newHashSet("bash");

        final JobRequest jobRequest1 = new JobRequest.Builder(JOB_NAME, JOB_USER, JOB_VERSION, "-c 'ls foo'",
                clusterCriteriaList, commandCriteria).withId(UUID.randomUUID().toString())
                        .withDisableLogArchival(true).build();

        jobClient.submitJob(jobRequest1);
        jobClient.waitForCompletion(jobRequest1.getId().orElseThrow(IllegalArgumentException::new), 60000, 5000);

        final InputStream inputStream1 = jobClient
                .getJobStderr(jobRequest1.getId().orElseThrow(IllegalArgumentException::new));
        final BufferedReader reader1 = new BufferedReader(new InputStreamReader(inputStream1, "UTF-8"));
        final StringBuilder sb = new StringBuilder();
        String line;
        while ((line = reader1.readLine()) != null) {
            sb.append(line);
        }

        reader1.close();
        inputStream1.close();

        Assert.assertEquals("ls: foo: No such file or directory", sb.toString());
    }

    /**
     * Method to test get stdout function.
     *
     * @throws Exception If there is a problem.
     */
    @Test(expected = GenieClientException.class)
    public void testPreconditionFailedException() throws Exception {

        createClusterAndCommandForTest();

        final List<ClusterCriteria> clusterCriteriaList = Lists
                .newArrayList(new ClusterCriteria(Sets.newHashSet("foo")));

        final Set<String> commandCriteria = Sets.newHashSet("bash");

        final JobRequest jobRequest1 = new JobRequest.Builder(JOB_NAME, JOB_USER, JOB_VERSION, "-c 'ls foo'",
                clusterCriteriaList, commandCriteria).withDisableLogArchival(true).build();

        jobClient.submitJob(jobRequest1);
    }

    /**
     * Helper method to create a cluster/command combination for all tests.
     *
     * @throws Exception If it fails to create the cluster/command combination.
     */
    private void createClusterAndCommandForTest() throws Exception {

        final Set<String> tags = Sets.newHashSet("laptop");

        final Cluster cluster = new Cluster.Builder(CLUSTER_NAME, "user", "1.0", ClusterStatus.UP).withTags(tags)
                .build();

        final String clusterId = clusterClient.createCluster(cluster);

        tags.clear();
        tags.add("bash");

        final Command command = new Command.Builder(COMMAND_NAME, "user", "version", CommandStatus.ACTIVE, "bash",
                1000).withTags(tags).build();

        final String commandId = commandClient.createCommand(command);

        clusterClient.addCommandsToCluster(clusterId, Lists.newArrayList(commandId));
    }
}