Java tutorial
/* * ProActive Parallel Suite(TM): * The Open Source library for parallel and distributed * Workflows & Scheduling, Orchestration, Cloud Automation * and Big Data Analysis on Enterprise Grids & Clouds. * * Copyright (c) 2007 - 2017 ActiveEon * Contact: contact@activeeon.com * * This library is free software: you can redistribute it and/or * modify it under the terms of the GNU Affero General Public License * as published by the Free Software Foundation: version 3 of * the License. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. * * If needed, contact us to obtain a release under GPL Version 2 or 3 * or a different license than the AGPL. */ package functionaltests.job.scheduling; import java.io.File; import java.net.URL; import java.util.ArrayList; import java.util.List; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.junit.After; import org.junit.Assert; import org.junit.Test; import org.ow2.proactive.scheduler.common.Scheduler; import org.ow2.proactive.scheduler.common.job.Job; import org.ow2.proactive.scheduler.common.job.JobId; import org.ow2.proactive.scheduler.common.job.JobPriority; import org.ow2.proactive.scheduler.common.job.TaskFlowJob; import org.ow2.proactive.scheduler.common.job.factories.JobFactory; import org.ow2.proactive.scheduler.common.task.JavaTask; import org.ow2.proactive.scheduler.common.task.ParallelEnvironment; import org.ow2.proactive.scheduler.common.task.TaskState; import org.ow2.proactive.scheduler.examples.EmptyTask; import org.ow2.proactive.scripting.SelectionScript; import com.google.common.collect.ImmutableMap; import functionaltests.utils.RMTHelper; import functionaltests.utils.SchedulerFunctionalTestNonForkModeWithRestart; /** * Several tests related to job scheduling priorities and starvation */ public class TestJobSchedulingStarvationAndPriority extends SchedulerFunctionalTestNonForkModeWithRestart { private static URL jobDescriptor = TestJobSchedulingStarvationAndPriority.class .getResource("/functionaltests/descriptors/Job_With_Replication.xml"); private static final String REPLICATE_TASK_NAME = "task2replicate"; private static final String REPLICATE_TASK_NAME_FILTER = REPLICATE_TASK_NAME + ".*"; /** * Tests that a starvation does not occur with selection scripts. A set of high-priority task with a selection script * returning false does not prevent a low priority task to run. * * @throws Exception */ @Test public void testStarvationSelectionScript() throws Exception { schedulerHelper.log("testStarvationSelectionScript"); Scheduler scheduler = schedulerHelper.getSchedulerInterface(); JobId jobIdLow; List<JobId> jobIds = new ArrayList<>(RMTHelper.DEFAULT_NODES_NUMBER); for (int i = 0; i < RMTHelper.DEFAULT_NODES_NUMBER; i++) { jobIds.add(scheduler.submit(createJobHighSelectFalse())); } jobIdLow = scheduler.submit(createJobLow()); schedulerHelper.waitForEventTaskRunning(jobIdLow, "taskLow"); for (int i = 0; i < RMTHelper.DEFAULT_NODES_NUMBER; i++) { schedulerHelper.killJob("" + jobIds.get(i)); } } /** * Tests that a starvation does not occur with multi-node tasks : * A high-priority task needing more nodes than available does not prevent a low priority task to execute. * * @throws Exception */ @Test public void testStarvationMultiNode() throws Exception { schedulerHelper.log("testStarvationMultiNode"); Scheduler scheduler = schedulerHelper.getSchedulerInterface(); JobId jobIdLow; JobId jobIdHigh; jobIdHigh = scheduler.submit(createJobHighMoreMultiNode()); jobIdLow = scheduler.submit(createJobLow()); schedulerHelper.waitForEventTaskRunning(jobIdLow, "taskLow"); schedulerHelper.killJob("" + jobIdHigh); } /** * Tests that all replicated tasks in a low priority job are executed AFTER replicated tasks in a high priority one * * @throws Exception */ @Test public void testJobPriorityStandard() throws Exception { schedulerHelper.log("testJobPriorityStandard"); testJobPriority(false); } /** * Tests that all replicated tasks in a low priority job are executed AFTER replicated tasks in a high priority one * * Additionally, add extra nodes during the run and ensure that new nodes are not picked by low priority tasks * * @throws Exception */ @Test public void testJobPriorityWithNewNodes() throws Exception { schedulerHelper.log("testJobPriorityWithNewNodes"); testJobPriority(true); } public void testJobPriority(boolean addNewNodes) throws Exception { Scheduler scheduler = schedulerHelper.getSchedulerInterface(); int nbNewNodes = 15; int nbRunsHigh = RMTHelper.DEFAULT_NODES_NUMBER * 2 + (addNewNodes ? nbNewNodes * 3 : 0); int nbRunsLow = RMTHelper.DEFAULT_NODES_NUMBER * 2 + (addNewNodes ? nbNewNodes * 2 : 0); String jobDescriptorPath = new File(jobDescriptor.toURI()).getAbsolutePath(); Job jobHigh = JobFactory.getFactory().createJob(jobDescriptorPath, ImmutableMap.of("RUNS", "" + nbRunsHigh)); jobHigh.setPriority(JobPriority.HIGH); JobId jobIdHigh = schedulerHelper.submitJob(jobHigh); schedulerHelper.waitForEventTaskRunning(jobIdHigh, REPLICATE_TASK_NAME); Job jobLow = JobFactory.getFactory().createJob(jobDescriptorPath, ImmutableMap.of("RUNS", "" + nbRunsLow)); jobLow.setPriority(JobPriority.LOW); JobId jobIdLow = schedulerHelper.submitJob(jobLow); if (addNewNodes) { schedulerHelper.addExtraNodes(nbNewNodes); } schedulerHelper.waitForEventJobFinished(jobIdHigh); schedulerHelper.waitForEventJobFinished(jobIdLow); Pair<Long, Long> minMaxHigh = computeMinMaxStartingTime(scheduler, jobIdHigh, REPLICATE_TASK_NAME_FILTER); Pair<Long, Long> minMaxLow = computeMinMaxStartingTime(scheduler, jobIdLow, REPLICATE_TASK_NAME_FILTER); Assert.assertTrue("Low Priority tasks min start time : " + minMaxLow.getLeft() + " should be greater than the max start time of high priority jobs : " + minMaxHigh.getRight(), minMaxLow.getLeft() > minMaxHigh.getRight()); } /** * Computes min start time and max start time for all tasks which match a given pattern * If a task in the set did not start, then the set max will be Long.MAX_VALUE */ Pair<Long, Long> computeMinMaxStartingTime(Scheduler scheduler, JobId jobId, String taskNameFilter) throws Exception { long min = Long.MAX_VALUE; long max = -1; for (TaskState state : scheduler.getJobState(jobId).getTasks()) { if (state.getName().matches(taskNameFilter)) { long startTime = state.getStartTime() > -1 ? state.getStartTime() : Long.MAX_VALUE; min = Math.min(min, startTime); max = Math.max(max, startTime); } } return new ImmutablePair<>(min, max); } @After public void releaseNodes() { try { schedulerHelper.removeExtraNodeSource(); } catch (Exception ignored) { } } /* * Job high priority with one task, task's selection script always returns 'false' so task can't * start */ private TaskFlowJob createJobHighSelectFalse() throws Exception { TaskFlowJob job = new TaskFlowJob(); job.setName(this.getClass().getSimpleName() + "_High_SelectFalse"); job.setPriority(JobPriority.HIGHEST); JavaTask javaTask = new JavaTask(); javaTask.setExecutableClassName(EmptyTask.class.getName()); javaTask.setName("taskSelectFalse"); SelectionScript selScript = new SelectionScript("selected = false;", "js"); javaTask.setSelectionScript(selScript); job.addTask(javaTask); return job; } /* * Job high priority with one task and with a required number of nodes greater than currently * available */ private TaskFlowJob createJobHighMoreMultiNode() throws Exception { TaskFlowJob job = new TaskFlowJob(); job.setName(this.getClass().getSimpleName() + "_High_MoreMultiNode"); job.setPriority(JobPriority.HIGHEST); JavaTask javaTask = new JavaTask(); javaTask.setExecutableClassName(EmptyTask.class.getName()); javaTask.setName("taskMoreMultiNode"); javaTask.setParallelEnvironment(new ParallelEnvironment(RMTHelper.DEFAULT_NODES_NUMBER + 1)); job.addTask(javaTask); return job; } /* * Job high priority with one task and high priority */ private TaskFlowJob createJobReplicate(int nbRun, JobPriority priority) throws Exception { String jobDescriptorPath = new File(jobDescriptor.toURI()).getAbsolutePath(); TaskFlowJob job = new TaskFlowJob(); job.setName(this.getClass().getSimpleName() + "_High"); job.setPriority(JobPriority.HIGHEST); JavaTask javaTask = new JavaTask(); javaTask.setExecutableClassName(EmptyTask.class.getName()); javaTask.setName("taskHigh"); job.addTask(javaTask); return job; } /* * Job high priority with one task and low priority */ private TaskFlowJob createJobLow() throws Exception { TaskFlowJob job = new TaskFlowJob(); job.setName(this.getClass().getSimpleName() + "_Low"); job.setPriority(JobPriority.LOW); JavaTask javaTask = new JavaTask(); javaTask.setExecutableClassName(EmptyTask.class.getName()); javaTask.setName("taskLow"); job.addTask(javaTask); return job; } }