Example usage for org.apache.hadoop.conf Configuration setInt

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration setInt.

Prototype

public void setInt(String name, int value) 

Source Link

Document

Set the value of the name property to an int.

Usage

From source file:com.cloudera.llama.am.impl.TestThrottleLlamaAM.java

License:Apache License

@Test
public void testThrottleQueueMaxOut() throws Exception {
    SingleQueueLlamaAM am = Mockito.mock(SingleQueueLlamaAM.class);

    Configuration conf = new Configuration(false);
    conf.setInt(ThrottleLlamaAM.MAX_PLACED_RESERVATIONS_KEY, 1);
    conf.setInt(ThrottleLlamaAM.MAX_QUEUED_RESERVATIONS_KEY, 2);
    ThrottleLlamaAM tAm = new ThrottleLlamaAM(conf, "q", am);
    try {//from  w ww.  java 2 s  . co  m
        tAm.start();
        Assert.assertEquals(0, tAm.getPlacedReservations());
        Assert.assertEquals(0, tAm.getQueuedReservations());

        Reservation r = TestUtils.createReservation(true);
        PlacedReservation pr = TestUtils.createPlacedReservation(r, PlacedReservation.Status.PENDING);

        Mockito.when(am.getReservation(Mockito.any(UUID.class))).thenReturn(pr);

        UUID id1 = tAm.reserve(r);
        Mockito.verify(am, VerificationModeFactory.times(1)).reserve(Mockito.any(UUID.class),
                Mockito.any(Reservation.class));
        Assert.assertEquals(1, tAm.getPlacedReservations());
        Assert.assertEquals(0, tAm.getQueuedReservations());

        UUID id2 = tAm.reserve(r);
        Mockito.verify(am, VerificationModeFactory.times(1)).reserve(Mockito.any(UUID.class),
                Mockito.any(Reservation.class));
        Assert.assertEquals(1, tAm.getPlacedReservations());
        Assert.assertEquals(1, tAm.getQueuedReservations());

        PlacedReservation pr2 = tAm.getReservation(id2);

        tAm.reserve(r);
        Mockito.verify(am, VerificationModeFactory.times(1)).reserve(Mockito.any(UUID.class),
                Mockito.any(Reservation.class));
        Assert.assertEquals(1, tAm.getPlacedReservations());
        Assert.assertEquals(2, tAm.getQueuedReservations());

        Assert.assertEquals(pr, tAm.getReservation(id1));
        Mockito.verify(am).getReservation(Mockito.eq(id1));
        Assert.assertEquals(pr2, tAm.getReservation(id2));

        try {
            tAm.reserve(r);
            Assert.fail();
        } catch (LlamaException ex) {
            Assert.assertEquals(ErrorCode.LLAMA_MAX_RESERVATIONS_FOR_QUEUE.getCode(), ex.getErrorCode());
        } catch (Throwable ex) {
            Assert.fail();
        }

        Assert.assertEquals(2, tAm.getQueuedReservations());

    } finally {
        tAm.stop();
    }
}

---

From source file:com.cloudera.llama.am.impl.TestThrottleLlamaAM.java

License:Apache License

@Test
public void testReleasePlaced() throws Exception {
    Reservation r = TestUtils.createReservation(true);
    PlacedReservation pr = TestUtils.createPlacedReservation(r, PlacedReservation.Status.PENDING);
    SingleQueueLlamaAM am = Mockito.mock(SingleQueueLlamaAM.class);

    Mockito.when(am.releaseReservation(Mockito.eq(pr.getHandle()), Mockito.eq(pr.getReservationId()),
            Mockito.eq(false))).thenReturn(pr);

    Configuration conf = new Configuration(false);
    conf.setInt(ThrottleLlamaAM.MAX_PLACED_RESERVATIONS_KEY, 1);
    conf.setInt(ThrottleLlamaAM.MAX_QUEUED_RESERVATIONS_KEY, 2);
    ThrottleLlamaAM tAm = new ThrottleLlamaAM(conf, "q", am);
    try {//from   w  w  w  . j  a v a 2  s .c om
        UUID id = pr.getReservationId();
        tAm.start();
        tAm.reserve(id, r);
        Assert.assertEquals(1, tAm.getPlacedReservations());
        Assert.assertEquals(0, tAm.getQueuedReservations());
        tAm.releaseReservation(pr.getHandle(), id, false);
        Mockito.verify(am, VerificationModeFactory.times(1)).releaseReservation(Mockito.any(UUID.class),
                Mockito.any(UUID.class), Mockito.anyBoolean());
        //simulation underlying AM release event
        ((PlacedReservationImpl) pr).setStatus(PlacedReservation.Status.RELEASED);
        tAm.onEvent(LlamaAMEventImpl.createEvent(true, pr));
        Assert.assertEquals(0, tAm.getPlacedReservations());
        Assert.assertEquals(0, tAm.getQueuedReservations());

        //forcing the no reservation path
        tAm.releaseReservation(pr.getHandle(), id, false);

    } finally {
        tAm.stop();
    }
}

---

From source file:com.cloudera.llama.am.impl.TestThrottleLlamaAM.java

License:Apache License

@Test
public void testReleaseQueued() throws Exception {
    Reservation r1 = TestUtils.createReservation(true);
    Reservation r2 = TestUtils.createReservation(true);
    PlacedReservation pr1 = TestUtils.createPlacedReservation(r1, PlacedReservation.Status.PENDING);
    PlacedReservation pr2 = TestUtils.createPlacedReservation(r2, PlacedReservation.Status.PENDING);

    SingleQueueLlamaAM am = Mockito.mock(SingleQueueLlamaAM.class);

    Mockito.when(am.releaseReservation(Mockito.eq(pr1.getHandle()), Mockito.eq(pr1.getReservationId()),
            Mockito.eq(false))).thenReturn(pr1);

    Configuration conf = new Configuration(false);
    conf.setInt(ThrottleLlamaAM.MAX_PLACED_RESERVATIONS_KEY, 1);
    conf.setInt(ThrottleLlamaAM.MAX_QUEUED_RESERVATIONS_KEY, 2);
    ThrottleLlamaAM tAm = new ThrottleLlamaAM(conf, "q", am);
    try {//from  w ww .j a  va2s . co m
        tAm.start();
        tAm.reserve(pr1.getReservationId(), r1);
        tAm.reserve(pr2.getReservationId(), r2);
        Assert.assertEquals(1, tAm.getPlacedReservations());
        Assert.assertEquals(1, tAm.getQueuedReservations());
        tAm.releaseReservation(pr2.getHandle(), pr2.getReservationId(), false);
        Mockito.verify(am, VerificationModeFactory.times(0)).releaseReservation(Mockito.any(UUID.class),
                Mockito.any(UUID.class), Mockito.anyBoolean());
        Assert.assertEquals(1, tAm.getPlacedReservations());
        Assert.assertEquals(0, tAm.getQueuedReservations());

        //forcing the no reservation path
        tAm.releaseReservation(pr2.getHandle(), pr2.getReservationId(), false);

    } finally {
        tAm.stop();
    }
}

---

From source file:com.cloudera.llama.am.impl.TestThrottleLlamaAM.java

License:Apache License

@Test
public void testPlaceFromQueueWhenPlacedReleased() throws Exception {
    Reservation r1 = TestUtils.createReservation(true);
    Reservation r2 = TestUtils.createReservation(true);
    PlacedReservation pr1 = TestUtils.createPlacedReservation(r1, PlacedReservation.Status.PENDING);
    PlacedReservation pr2 = TestUtils.createPlacedReservation(r2, PlacedReservation.Status.PENDING);

    SingleQueueLlamaAM am = Mockito.mock(SingleQueueLlamaAM.class);

    Mockito.when(am.releaseReservation(Mockito.eq(pr1.getHandle()), Mockito.eq(pr1.getReservationId()),
            Mockito.eq(false))).thenReturn(pr1);

    Configuration conf = new Configuration(false);
    conf.setInt(ThrottleLlamaAM.MAX_PLACED_RESERVATIONS_KEY, 1);
    conf.setInt(ThrottleLlamaAM.MAX_QUEUED_RESERVATIONS_KEY, 2);
    ThrottleLlamaAM tAm = new ThrottleLlamaAM(conf, "q", am);
    try {// ww w. j  a v  a 2 s .  c om
        tAm.start();
        tAm.reserve(pr1.getReservationId(), r1);
        tAm.reserve(pr2.getReservationId(), r2);
        Assert.assertEquals(1, tAm.getPlacedReservations());
        Assert.assertEquals(1, tAm.getQueuedReservations());
        tAm.releaseReservation(pr1.getHandle(), pr1.getReservationId(), false);
        //simulation underlying AM release event
        ((PlacedReservationImpl) pr1).setStatus(PlacedReservation.Status.RELEASED);
        tAm.onEvent(LlamaAMEventImpl.createEvent(true, pr1));
        manualClock.increment(1001);
        Thread.sleep(100);

        Assert.assertEquals(1, tAm.getPlacedReservations());
        Assert.assertEquals(0, tAm.getQueuedReservations());
        Mockito.verify(am, VerificationModeFactory.times(2)).reserve(Mockito.any(UUID.class),
                Mockito.any(Reservation.class));
    } finally {
        tAm.stop();
    }
}

---

From source file:com.cloudera.llama.am.impl.TestThrottleLlamaAM.java

License:Apache License

@Test
public void testPlaceFromQueueWhenPlacedEndsBecauseOfRM() throws Exception {
    Reservation r1 = TestUtils.createReservation(true);
    Reservation r2 = TestUtils.createReservation(true);
    PlacedReservation pr1 = TestUtils.createPlacedReservation(r1, PlacedReservation.Status.PENDING);
    PlacedReservation pr2 = TestUtils.createPlacedReservation(r2, PlacedReservation.Status.PENDING);

    SingleQueueLlamaAM am = Mockito.mock(SingleQueueLlamaAM.class);

    Mockito.when(am.releaseReservation(Mockito.eq(pr1.getHandle()), Mockito.eq(pr1.getReservationId()),
            Mockito.eq(false))).thenReturn(pr1);

    Configuration conf = new Configuration(false);
    conf.setInt(ThrottleLlamaAM.MAX_PLACED_RESERVATIONS_KEY, 1);
    conf.setInt(ThrottleLlamaAM.MAX_QUEUED_RESERVATIONS_KEY, 2);
    ThrottleLlamaAM tAm = new ThrottleLlamaAM(conf, "q", am);
    try {/*from   w w  w. java2 s. c  om*/
        tAm.start();
        tAm.reserve(pr1.getReservationId(), r1);
        tAm.reserve(pr2.getReservationId(), r2);
        Assert.assertEquals(1, tAm.getPlacedReservations());
        Assert.assertEquals(1, tAm.getQueuedReservations());

        LlamaAMEventImpl event = new LlamaAMEventImpl();
        ((PlacedReservationImpl) pr1).setStatus(PlacedReservation.Status.ALLOCATED);
        event.addReservation(pr1);
        tAm.onEvent(event);
        manualClock.increment(1001);
        Thread.sleep(100);

        Assert.assertEquals(1, tAm.getPlacedReservations());
        Assert.assertEquals(1, tAm.getQueuedReservations());

        event = new LlamaAMEventImpl();
        ((PlacedReservationImpl) pr1).setStatus(PlacedReservation.Status.PREEMPTED);
        event.addReservation(pr1);
        tAm.onEvent(event);
        manualClock.increment(1001);
        Thread.sleep(100);

        Assert.assertEquals(1, tAm.getPlacedReservations());
        Assert.assertEquals(0, tAm.getQueuedReservations());
        Mockito.verify(am, VerificationModeFactory.times(2)).reserve(Mockito.any(UUID.class),
                Mockito.any(Reservation.class));
    } finally {
        tAm.stop();
    }
}

---

From source file:com.cloudera.llama.am.impl.TestThrottleLlamaAM.java

License:Apache License

@Test
public void testReleaseForHandle() throws Exception {
    Reservation r1 = TestUtils.createReservation(true);
    Reservation r2 = TestUtils.createReservation(true);
    PlacedReservation pr1 = TestUtils.createPlacedReservation(r1, PlacedReservation.Status.PENDING);
    PlacedReservation pr2 = TestUtils.createPlacedReservation(r2, PlacedReservation.Status.PENDING);

    UUID handle = UUID.randomUUID();
    Reservation r3 = TestUtils.createReservation(handle, 1, false);
    Reservation r4 = TestUtils.createReservation(handle, 1, false);
    PlacedReservation pr3 = TestUtils.createPlacedReservation(r3, PlacedReservation.Status.PENDING);
    PlacedReservation pr4 = TestUtils.createPlacedReservation(r4, PlacedReservation.Status.PENDING);

    SingleQueueLlamaAM am = Mockito.mock(SingleQueueLlamaAM.class);

    Mockito.when(am.releaseReservation(Mockito.eq(pr1.getHandle()), Mockito.eq(pr1.getReservationId()),
            Mockito.eq(false))).thenReturn(pr1);

    Mockito.when(am.releaseReservationsForHandle(Mockito.eq(pr3.getHandle()), Mockito.eq(false)))
            .thenReturn(Arrays.asList(pr3));

    //config defaults

    Configuration conf = new Configuration(false);
    conf.setInt(ThrottleLlamaAM.MAX_PLACED_RESERVATIONS_KEY, 2);
    conf.setInt(ThrottleLlamaAM.MAX_QUEUED_RESERVATIONS_KEY, 2);
    ThrottleLlamaAM tAm = new ThrottleLlamaAM(conf, "q", am);
    try {/*from ww w . jav a2s  .  com*/
        tAm.start();
        tAm.reserve(pr1.getReservationId(), r1);
        tAm.reserve(pr3.getReservationId(), r3);
        tAm.reserve(pr2.getReservationId(), r2);
        tAm.reserve(pr4.getReservationId(), r4);
        Assert.assertEquals(2, tAm.getPlacedReservations());
        Assert.assertEquals(2, tAm.getQueuedReservations());
        List<PlacedReservation> released = tAm.releaseReservationsForHandle(handle, false);
        //simulation underlying AM release event
        ((PlacedReservationImpl) pr3).setStatus(PlacedReservation.Status.RELEASED);
        tAm.onEvent(LlamaAMEventImpl.createEvent(true, pr3));

        Assert.assertTrue(released.contains(pr3));
        Assert.assertTrue(released.contains(pr4));
        manualClock.increment(1001);
        Thread.sleep(100);
        Assert.assertEquals(2, tAm.getPlacedReservations());
        Assert.assertEquals(0, tAm.getQueuedReservations());
        Mockito.verify(am, VerificationModeFactory.times(3)).reserve(Mockito.any(UUID.class),
                Mockito.any(Reservation.class));
    } finally {
        tAm.stop();
    }
}

---

From source file:com.cloudera.llama.am.impl.TestThrottleLlamaAM.java

License:Apache License

@Test
public void testReleaseForQueue() throws Exception {
    Reservation r1 = TestUtils.createReservation(true);
    Reservation r2 = TestUtils.createReservation(true);
    PlacedReservation pr1 = TestUtils.createPlacedReservation(r1, PlacedReservation.Status.PENDING);
    PlacedReservation pr2 = TestUtils.createPlacedReservation(r2, PlacedReservation.Status.PENDING);

    SingleQueueLlamaAM am = Mockito.mock(SingleQueueLlamaAM.class);

    Mockito.when(am.releaseReservationsForQueue(Mockito.anyString(), Mockito.eq(false)))
            .thenReturn(Arrays.asList(pr1));

    Configuration conf = new Configuration(false);
    conf.setInt(ThrottleLlamaAM.MAX_PLACED_RESERVATIONS_KEY, 1);
    conf.setInt(ThrottleLlamaAM.MAX_QUEUED_RESERVATIONS_KEY, 1);
    ThrottleLlamaAM tAm = new ThrottleLlamaAM(conf, "q", am);
    try {/*  ww w .  ja v  a 2  s . co  m*/
        tAm.start();
        tAm.reserve(pr1.getReservationId(), r1);
        tAm.reserve(pr2.getReservationId(), r2);
        Assert.assertEquals(1, tAm.getPlacedReservations());
        Assert.assertEquals(1, tAm.getQueuedReservations());
        List<PlacedReservation> released = tAm.releaseReservationsForQueue("q", false);
        //simulation underlying AM release event
        ((PlacedReservationImpl) pr1).setStatus(PlacedReservation.Status.RELEASED);
        tAm.onEvent(LlamaAMEventImpl.createEvent(true, pr1));
        Assert.assertEquals(2, released.size());
        Assert.assertTrue(released.contains(pr1));
        Assert.assertTrue(released.contains(pr2));
        manualClock.increment(1001);
        Thread.sleep(100);
        Assert.assertEquals(0, tAm.getPlacedReservations());
        Assert.assertEquals(0, tAm.getQueuedReservations());
        Mockito.verify(am, VerificationModeFactory.times(1)).reserve(Mockito.any(UUID.class),
                Mockito.any(Reservation.class));

    } finally {
        tAm.stop();
    }
}

---

From source file:com.cloudera.llama.am.MiniLlama.java

License:Apache License

public static Configuration createMiniLlamaConf(Configuration conf, int nodes) {
    ParamChecker.notNull(conf, "conf");
    ParamChecker.greaterThan(nodes, 0, "nodes");
    conf.set(ServerConfiguration.CONFIG_DIR_KEY, "");
    conf.setIfUnset(LlamaAM.RM_CONNECTOR_CLASS_KEY, YarnRMConnector.class.getName());
    conf.setInt(MINI_CLUSTER_NODES_KEY, nodes);
    conf.setIfUnset(S_CONF.getPropertyName(ServerConfiguration.SERVER_ADDRESS_KEY), "localhost:0");
    conf.setIfUnset(S_CONF.getPropertyName(ServerConfiguration.SERVER_ADMIN_ADDRESS_KEY), "localhost:0");
    conf.setIfUnset(S_CONF.getPropertyName(ServerConfiguration.HTTP_ADDRESS_KEY), "localhost:0");
    conf.setBoolean(YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, true);
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 0);
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0);

    conf.set(YarnRMConnector.HADOOP_USER_NAME_KEY, System.getProperty("user.name"));
    return conf;//from  w  w  w  .j  av a 2  s .co  m
}

---

From source file:com.cloudera.llama.am.MiniLlama.java

License:Apache License

private Configuration startMiniHadoop() throws Exception {
    int clusterNodes = getConf().getInt(MINI_CLUSTER_NODES_KEY, 1);
    if (System.getProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA) == null) {
        String testBuildData = new File("target").getAbsolutePath();
        System.setProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA, testBuildData);
    }/* w  w w .  j a  va2  s.  c om*/
    //to trigger hdfs-site.xml registration as default resource
    new HdfsConfiguration();
    Configuration conf = new YarnConfiguration();
    String llamaProxyUser = System.getProperty("user.name");
    conf.set("hadoop.security.authentication", "simple");
    conf.set("hadoop.proxyuser." + llamaProxyUser + ".hosts", "*");
    conf.set("hadoop.proxyuser." + llamaProxyUser + ".groups", "*");
    String[] userGroups = new String[] { "g" };
    UserGroupInformation.createUserForTesting(llamaProxyUser, userGroups);

    int hdfsPort = 0;
    String fsUri = conf.get("fs.defaultFS");
    if (fsUri != null && !fsUri.equals("file:///")) {
        int i = fsUri.lastIndexOf(":");
        if (i > -1) {
            try {
                hdfsPort = Integer.parseInt(fsUri.substring(i + 1));
            } catch (Exception ex) {
                throw new RuntimeException(
                        "Could not parse port from Hadoop's " + "'fs.defaultFS property: " + fsUri);
            }
        }
    }
    miniHdfs = new MiniDFSCluster(hdfsPort, conf, clusterNodes, !skipDfsFormat, true, null, null);
    miniHdfs.waitActive();
    conf = miniHdfs.getConfiguration(0);
    miniYarn = new MiniYARNCluster("minillama", clusterNodes, 1, 1);
    conf.setBoolean(YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, true);
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 0);
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0);

    miniYarn.init(conf);
    miniYarn.start();
    conf = miniYarn.getConfig();

    ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
    return conf;
}

---

From source file:com.cloudera.llama.am.mock.TestLlamaAMWithMock.java

License:Apache License

protected Configuration getConfiguration() {
    Configuration conf = new Configuration(false);
    conf.set("llama.am.mock.queues", "q1,q2");
    conf.set("llama.am.mock.events.min.wait.ms", "10");
    conf.set("llama.am.mock.events.max.wait.ms", "10");
    conf.set("llama.am.mock.nodes", "h0,h1,h2,h3");
    conf.set(LlamaAM.CORE_QUEUES_KEY, "q1");
    conf.set(LlamaAM.RM_CONNECTOR_CLASS_KEY, MockRMConnector.class.getName());
    conf.setInt(ThrottleLlamaAM.MAX_PLACED_RESERVATIONS_KEY, 1000000);
    conf.setInt(ThrottleLlamaAM.MAX_QUEUED_RESERVATIONS_KEY, 1000000);
    return conf;/*w  ww .j a  v a 2  s.c  o  m*/
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »