Example usage for org.apache.hadoop.conf Configuration setInt

List of usage examples for org.apache.hadoop.conf Configuration setInt

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. setInt

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration setInt.

Prototype

public void setInt(String name, int value)

Source Link

Document

Set the value of the name property to an int.

Usage

From source file:com.cloudera.llama.am.TestLlamaAMThriftServer.java

License:Apache License

protected Configuration createLlamaConfiguration() throws Exception {
    Configuration conf = new Configuration(false);
    conf.set(ServerConfiguration.CONFIG_DIR_KEY, TestAbstractMain.createTestDir());

    conf.setClass(LlamaAM.RM_CONNECTOR_CLASS_KEY, MockRMConnector.class, RMConnector.class);
    conf.set(LlamaAM.CORE_QUEUES_KEY, "root.q1,root.q2");
    conf.set(MockRMConnector.QUEUES_KEY, "root.q1,root.q2");
    String nodesKey = "" + MockLlamaAMFlags.ALLOCATE + "n1";
    conf.set(MockRMConnector.NODES_KEY, nodesKey);
    conf.setInt(MockRMConnector.EVENTS_MIN_WAIT_KEY, 5);
    conf.setInt(MockRMConnector.EVENTS_MAX_WAIT_KEY, 10);

    conf.set(sConf.getPropertyName(ServerConfiguration.SERVER_ADDRESS_KEY), "localhost:0");
    conf.set(sConf.getPropertyName(ServerConfiguration.SERVER_ADMIN_ADDRESS_KEY), "localhost:0");
    conf.set(sConf.getPropertyName(ServerConfiguration.HTTP_ADDRESS_KEY), "localhost:0");
    return conf;//  w  ww  .  ja v  a 2s.c om
}

From source file:com.cloudera.llama.am.TestLlamaAMThriftServer.java

License:Apache License

@Test
public void testDiscardReservationsOnMissingClient() throws Exception {
    final LlamaAMServer server = new LlamaHAServer();
    final NotificationEndPoint callbackServer = new NotificationEndPoint();
    try {/*from   w  ww. j  a v a  2  s .co  m*/
        callbackServer.setConf(createCallbackConfiguration());
        callbackServer.start();

        callbackServer.delayResponse = 250;

        Configuration conf = createLlamaConfiguration();
        conf.setInt(sConf.getPropertyName(ServerConfiguration.CLIENT_NOTIFIER_HEARTBEAT_KEY), 10000);
        conf.setInt(sConf.getPropertyName(ServerConfiguration.CLIENT_NOTIFIER_RETRY_INTERVAL_KEY), 200);
        conf.setInt(sConf.getPropertyName(ServerConfiguration.CLIENT_NOTIFIER_MAX_RETRIES_KEY), 0);
        conf.setInt(sConf.getPropertyName(ServerConfiguration.TRANSPORT_TIMEOUT_KEY), 200);
        server.setConf(conf);

        server.start();

        Subject.doAs(getClientSubject(), new PrivilegedExceptionAction<Object>() {
            @Override
            public Object run() throws Exception {
                com.cloudera.llama.thrift.LlamaAMService.Client client = createClient(server);

                TLlamaAMRegisterRequest trReq = new TLlamaAMRegisterRequest();
                trReq.setVersion(TLlamaServiceVersion.V1);
                trReq.setClient_id(TypeUtils.toTUniqueId(UUID.randomUUID()));
                TNetworkAddress tAddress = new TNetworkAddress();
                tAddress.setHostname(callbackServer.getAddressHost());
                tAddress.setPort(callbackServer.getAddressPort());
                trReq.setNotification_callback_service(tAddress);

                //register
                TLlamaAMRegisterResponse trRes = client.Register(trReq);
                Assert.assertEquals(TStatusCode.OK, trRes.getStatus().getStatus_code());

                //make reservation
                TLlamaAMReservationRequest tresReq = new TLlamaAMReservationRequest();
                tresReq.setVersion(TLlamaServiceVersion.V1);
                tresReq.setAm_handle(trRes.getAm_handle());
                tresReq.setUser(getUserName());
                tresReq.setQueue("q1");
                TResource tResource = new TResource();
                tResource.setClient_resource_id(TypeUtils.toTUniqueId(UUID.randomUUID()));
                tresReq.setReservation_id(TypeUtils.toTUniqueId(UUID.randomUUID()));
                tResource.setAskedLocation(MockLlamaAMFlags.ALLOCATE + "n1");
                tResource.setV_cpu_cores((short) 1);
                tResource.setMemory_mb(1024);
                tResource.setEnforcement(TLocationEnforcement.MUST);
                tresReq.setResources(Arrays.asList(tResource));
                tresReq.setGang(true);
                TLlamaAMReservationResponse tresRes = client.Reserve(tresReq);
                Assert.assertEquals(TStatusCode.OK, tresRes.getStatus().getStatus_code());

                Thread.sleep(250); //extra 50sec
                Assert.assertEquals(0, callbackServer.notifications.size());
                callbackServer.delayResponse = 0;

                //release
                client = createClient(server);
                TLlamaAMReleaseRequest trelReq = new TLlamaAMReleaseRequest();
                trelReq.setVersion(TLlamaServiceVersion.V1);
                trelReq.setAm_handle(trRes.getAm_handle());
                trelReq.setReservation_id(tresRes.getReservation_id());
                TLlamaAMReleaseResponse trelRes = client.Release(trelReq);
                Assert.assertEquals(TStatusCode.REQUEST_ERROR, trelRes.getStatus().getStatus_code());
                Assert.assertEquals(ErrorCode.CLIENT_UNKNOWN_HANDLE.getCode(),
                        trelRes.getStatus().getError_code());
                return null;
            }
        });
    } finally {
        server.stop();
        callbackServer.stop();
    }
}

From source file:com.cloudera.llama.am.TestLlamaHAServer.java

License:Apache License

@Before
public void setup() {
    Configuration conf = new Configuration(false);
    conf.set(ServerConfiguration.CONFIG_DIR_KEY, TestAbstractMain.createTestDir());
    conf.setClass(LlamaAM.RM_CONNECTOR_CLASS_KEY, MockRMConnector.class, RMConnector.class);
    conf.set(LlamaAM.CORE_QUEUES_KEY, "root.q1,root.q2");
    conf.set(MockRMConnector.QUEUES_KEY, "root.q1,root.q2");
    String nodesKey = "" + MockLlamaAMFlags.ALLOCATE + "n1";
    conf.set(MockRMConnector.NODES_KEY, nodesKey);
    conf.setInt(MockRMConnector.EVENTS_MIN_WAIT_KEY, 5);
    conf.setInt(MockRMConnector.EVENTS_MAX_WAIT_KEY, 10);

    ServerConfiguration sConf = new AMServerConfiguration(conf);
    conf.set(sConf.getPropertyName(ServerConfiguration.SERVER_ADDRESS_KEY), "localhost:0");
    conf.set(sConf.getPropertyName(ServerConfiguration.SERVER_ADMIN_ADDRESS_KEY), "localhost:0");
    conf.set(sConf.getPropertyName(ServerConfiguration.HTTP_ADDRESS_KEY), "localhost:0");

    conf.setBoolean(HAServerConfiguration.HA_ENABLED, true);
    server = new LlamaHAServer();
    server.setConf(conf);/* w  ww .  jav  a 2s.  com*/
}

From source file:com.cloudera.llama.am.yarn.TestLlamaAMWithYarn.java

License:Apache License

private Configuration createMiniYarnConfig(boolean usePortInName) throws Exception {
    Configuration conf = new YarnConfiguration();
    conf.set("yarn.scheduler.fair.allocation.file", "test-fair-scheduler.xml");
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0);
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 0);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, FairScheduler.class);

    //proxy user config
    String llamaProxyUser = System.getProperty("user.name");
    conf.set("hadoop.security.authentication", "simple");
    conf.set("hadoop.proxyuser." + llamaProxyUser + ".hosts", "*");
    conf.set("hadoop.proxyuser." + llamaProxyUser + ".groups", "*");
    String[] userGroups = new String[] { "g" };
    UserGroupInformation.createUserForTesting(llamaProxyUser, userGroups);
    conf.setBoolean(YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, usePortInName);
    return conf;/*from  ww w.ja  v  a  2s .  co m*/
}

From source file:com.cloudera.llama.am.yarn.TestLlamaAMWithYarn.java

License:Apache License

protected static Configuration getLlamaConfiguration() {
    Configuration conf = new Configuration(false);
    conf.set(LlamaAM.CORE_QUEUES_KEY, "root.queue1,root.queue2");
    conf.set(LlamaAM.RM_CONNECTOR_CLASS_KEY, YarnRMConnector.class.getName());
    conf.setInt(YarnRMConnector.HEARTBEAT_INTERVAL_KEY, 50);
    for (Map.Entry entry : miniYarn.getConfig()) {
        conf.set((String) entry.getKey(), (String) entry.getValue());
    }/* w  w  w.j  a  v  a  2 s. c  om*/
    conf.set(YarnRMConnector.HADOOP_USER_NAME_KEY, System.getProperty("user.name"));
    return conf;
}

From source file:com.cloudera.llama.am.yarn.TestLlamaAMWithYarn.java

License:Apache License

@Test
public void testClusterNMChanges() throws Exception {
    try {//w ww . java 2s  .c o  m
        Configuration conf = createMiniYarnConfig(true);
        conf.setInt(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 1000);
        startYarn(conf, 2);
        LlamaAM llama = LlamaAM.create(getLlamaConfiguration());
        MyListener listener = new MyListener();
        try {
            llama.start();
            llama.addListener(listener);
            List<NodeInfo> nodes = llama.getNodes();
            Assert.assertEquals(2, nodes.size());
            miniYarn.getNodeManager(0).stop();
            long startTime = System.currentTimeMillis();
            while (llama.getNodes().size() != 1 && System.currentTimeMillis() - startTime < 10000) {
                Thread.sleep(100);
            }
            nodes = llama.getNodes();
            Assert.assertEquals(1, nodes.size());
        } finally {
            llama.stop();
        }
    } finally {
        stopYarn();
    }
}

From source file:com.cloudera.llama.am.yarn.TestLlamaAMWithYarn.java

License:Apache License

@Test
public void testResourceRejections() throws Exception {
    try {/*from  w  w  w.  j  a v a2s.  co  m*/
        Configuration conf = createMiniYarnConfig(true);
        conf.setInt(YarnConfiguration.NM_VCORES, 1);
        conf.setInt(YarnConfiguration.NM_PMEM_MB, 4096);
        conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, 2);
        conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 5020);
        startYarn(conf, 1);
        Configuration llamaConf = getLlamaConfiguration();
        llamaConf.setBoolean(LlamaAM.NORMALIZING_ENABLED_KEY, false);
        LlamaAM llama = LlamaAM.create(llamaConf);
        try {
            llama.start();
            List<NodeInfo> nodes = llama.getNodes();

            //invalid node
            try {
                Resource r = TestUtils.createResource("xyz:-1", Resource.Locality.MUST, 1, 4096);
                llama.reserve(TestUtils.createReservation(UUID.randomUUID(), "u", "root.queue1",
                        Arrays.asList(r), true));
                Assert.fail();
            } catch (LlamaException ex) {
                //NOP
            }

            //over max cpus
            try {
                Resource r = TestUtils.createResource(nodes.get(0).getLocation(), Resource.Locality.MUST, 3,
                        4096);
                llama.reserve(TestUtils.createReservation(UUID.randomUUID(), "u", "root.queue1",
                        Arrays.asList(r), true));
                Assert.fail();
            } catch (LlamaException ex) {
                //NOP
            }

            //over max memory
            try {
                Resource r = TestUtils.createResource(nodes.get(0).getLocation(), Resource.Locality.MUST, 1,
                        4097);
                llama.reserve(TestUtils.createReservation(UUID.randomUUID(), "u", "root.queue1",
                        Arrays.asList(r), true));
                Assert.fail();
            } catch (LlamaException ex) {
                //NOP
            }

            //over node cpus
            try {
                Resource r = TestUtils.createResource(nodes.get(0).getLocation(), Resource.Locality.MUST, 2,
                        4096);
                llama.reserve(TestUtils.createReservation(UUID.randomUUID(), "u", "root.queue1",
                        Arrays.asList(r), true));
                Assert.fail();
            } catch (LlamaException ex) {
                //NOP
            }

            //over node memory
            try {
                Resource r = TestUtils.createResource(nodes.get(0).getLocation(), Resource.Locality.MUST, 1,
                        5021);
                llama.reserve(TestUtils.createReservation(UUID.randomUUID(), "u", "root.queue1",
                        Arrays.asList(r), true));
                Assert.fail();
            } catch (LlamaException ex) {
                //NOP
            }

        } finally {
            llama.stop();
        }
    } finally {
        stopYarn();
    }
}

From source file:com.cloudera.llama.server.TestClientNotifier.java

License:Apache License

@Test
public void testHeartbeats() throws Exception {
    Configuration conf = new Configuration(false);
    conf.setInt(nsConf.getPropertyName(ServerConfiguration.CLIENT_NOTIFIER_HEARTBEAT_KEY), 300);
    ServerConfiguration sConf = new NSServerConfiguration(conf);
    UUID clientId = UUID.randomUUID();
    UUID handle = UUID.randomUUID();
    MyClientRegistry cr = new MyClientRegistry(sConf, clientId, handle, notificationServer.getAddressHost(),
            notificationServer.getAddressPort());
    ClientNotifier cn = new ClientNotifier(sConf, new HostnameOnlyNodeMapper(), cr, null);
    try {/*from www .  j  av  a2s  . co  m*/
        cn.start();
        cn.registerClientForHeartbeats(handle);
        Assert.assertEquals(0, notificationServer.notifications.size());
        Thread.sleep(350); //adding 50ms extra
        Assert.assertEquals(1, notificationServer.notifications.size());
        Assert.assertTrue(notificationServer.notifications.get(0).isHeartbeat());
        notificationServer.notifications.clear();
        Thread.sleep(350); //adding 50ms extra
        Assert.assertEquals(1, notificationServer.notifications.size());
        Assert.assertTrue(notificationServer.notifications.get(0).isHeartbeat());
    } finally {
        cn.stop();
    }
}

From source file:com.cloudera.llama.server.TestClientNotifier.java

License:Apache License

@Test
public void testNotificationAndHeartbeatReset() throws Exception {
    Configuration conf = new Configuration(false);
    conf.setInt(nsConf.getPropertyName(ServerConfiguration.CLIENT_NOTIFIER_HEARTBEAT_KEY), 300);
    ServerConfiguration sConf = new NSServerConfiguration(conf);
    UUID clientId = UUID.randomUUID();
    UUID handle = UUID.randomUUID();
    MyClientRegistry cr = new MyClientRegistry(sConf, clientId, handle, notificationServer.getAddressHost(),
            notificationServer.getAddressPort());
    ClientNotifier cn = new ClientNotifier(sConf, new HostnameOnlyNodeMapper(), cr, null);
    try {/*from  w ww . j a va 2s  . c om*/
        cn.start();
        cn.registerClientForHeartbeats(handle);
        LlamaAMEventImpl event = new LlamaAMEventImpl();
        event.addReservation(TestUtils.createPlacedReservation(TestUtils.createReservation(true),
                PlacedReservation.Status.ALLOCATED));
        Thread.sleep(100);
        cn.onEvent(event);
        Assert.assertEquals(0, notificationServer.notifications.size());
        Thread.sleep(180);
        Assert.assertEquals(1, notificationServer.notifications.size());
        Assert.assertFalse(notificationServer.notifications.get(0).isHeartbeat());
        notificationServer.notifications.clear();
        Thread.sleep(170); //adding 50ms extra
        Assert.assertEquals(1, notificationServer.notifications.size());
        Assert.assertTrue(notificationServer.notifications.get(0).isHeartbeat());
    } finally {
        cn.stop();
    }
}

From source file:com.cloudera.llama.server.TestClientNotifier.java

License:Apache License

@Test
public void testRetriesAndMaxFailures() throws Exception {
    Configuration conf = new Configuration(false);
    conf.setInt(nsConf.getPropertyName(ServerConfiguration.CLIENT_NOTIFIER_HEARTBEAT_KEY), 10);
    conf.setInt(nsConf.getPropertyName(ServerConfiguration.CLIENT_NOTIFIER_RETRY_INTERVAL_KEY), 10);
    conf.setInt(nsConf.getPropertyName(ServerConfiguration.CLIENT_NOTIFIER_MAX_RETRIES_KEY), 2);
    conf.setInt(nsConf.getPropertyName(ServerConfiguration.TRANSPORT_TIMEOUT_KEY), 10);
    ServerConfiguration sConf = new NSServerConfiguration(conf);
    UUID clientId = UUID.randomUUID();
    UUID handle = UUID.randomUUID();
    MyClientRegistry cr = new MyClientRegistry(sConf, clientId, handle, notificationServer.getAddressHost(), 0);
    ClientNotifier cn = new ClientNotifier(sConf, new HostnameOnlyNodeMapper(), cr, null);
    try {/* w w  w .  j  a  va2 s  .c om*/
        cn.start();
        cn.registerClientForHeartbeats(handle);
        Thread.sleep(100); //adding 70ms extra
        Assert.assertEquals(3, cr.clientCallerCalls.get());
        Assert.assertTrue(cr.maxFailures);
    } finally {
        cn.stop();
    }
}