Example usage for org.apache.hadoop.conf Configuration getInt

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getInt.

Prototype

public int getInt(String name, int defaultValue)

Source Link

Document

Get the value of the name property as an int.

Usage

From source file:com.quantcast.qfs.hadoop.Qfs.java

License:Apache License

public Qfs(URI uri, Configuration conf) throws IOException, URISyntaxException {
    super(uri, kUriScheme, kUriAuthorityNeededFlag, kDefaultPort);
    final String scheme = uri.getScheme();
    if (!scheme.equalsIgnoreCase(kUriScheme)) {
        throw new IllegalArgumentException("requested URI scheme: " + scheme + " is not for QFS");
    }//from w ww. j a v  a 2s .  c o m
    if (uri.getHost() == null) {
        this.qfsImpl = new QFSImpl(conf.get("fs.qfs.metaServerHost", ""),
                conf.getInt("fs.qfs.metaServerPort", -1), getStatistics(), conf);
    } else {
        this.qfsImpl = new QFSImpl(uri.getHost(), uri.getPort(), getStatistics(), conf);
    }
    this.qfs = new QuantcastFileSystem2(this.qfsImpl, uri);
}

From source file:com.quantcast.qfs.hadoop.QuantcastFileSystem.java

License:Apache License

public void initialize(URI uri, Configuration conf) throws IOException {
    super.initialize(uri, conf);
    setConf(conf);//from   ww w .j  a  v a 2s.c  o  m

    try {
        if (qfsImpl == null) {
            if (uri.getHost() == null) {
                qfsImpl = createIFSImpl(conf.get("fs.qfs.metaServerHost", ""),
                        conf.getInt("fs.qfs.metaServerPort", -1), statistics, conf);
            } else {
                qfsImpl = createIFSImpl(uri.getHost(), uri.getPort(), statistics, conf);
            }
        }

        this.localFs = FileSystem.getLocal(conf);
        this.uri = URI
                .create(uri.getScheme() + "://" + (uri.getAuthority() == null ? "/" : uri.getAuthority()));
        this.workingDir = new Path("/user", System.getProperty("user.name")).makeQualified(uri, null);
        this.qfsImpl.setUMask(FsPermission.getUMask(conf).toShort());
    } catch (Exception e) {
        throw new IOException("Unable to initialize QFS using uri " + uri);
    }
}

From source file:com.qubole.rubix.hadoop1.Hadoop1ClusterManager.java

License:Apache License

@Override
public void initialize(Configuration conf) {
    super.initialize(conf);
    nnPort = conf.getInt(nnPortConf, nnPort);

    nodesSupplier = Suppliers.memoizeWithExpiration(new Supplier<List<String>>() {
        @Override/*from   ww  w . j  a va 2  s  . c  o  m*/
        public List<String> get() {
            if (!isMaster) {
                // First time all nodes start assuming themselves as master and down the line figure out their role
                // Next time onwards, only master will be fetching the list of nodes
                return ImmutableList.of();
            }

            HttpClient httpclient = new HttpClient();
            HttpMethod method = new GetMethod(
                    "http://localhost:" + nnPort + "/dfsnodelist.jsp?whatNodes=LIVE&status=NORMAL");
            int sc;
            try {
                sc = httpclient.executeMethod(method);
            } catch (IOException e) {
                // not reachable => worker
                sc = -1;
            }

            if (sc != 200) {
                LOG.debug("Could not reach dfsnodelist.jsp, setting worker role");
                isMaster = false;
                return ImmutableList.of();
            }

            LOG.debug("Reached dfsnodelist.jsp, setting master role");
            isMaster = true;
            String html;
            try {
                byte[] buf = method.getResponseBody();
                html = new String(buf);
            } catch (IOException e) {
                throw Throwables.propagate(e);
            }

            List<String> nodes = extractNodes(html);
            return nodes;
        }
    }, 10, TimeUnit.SECONDS);
    nodesSupplier.get();
}

From source file:com.qubole.rubix.presto.PrestoClusterManager.java

License:Apache License

@Override
public void initialize(Configuration conf) {
    super.initialize(conf);
    this.serverPort = conf.getInt(serverPortConf, serverPort);

    nodesSupplier = Suppliers.memoizeWithExpiration(new Supplier<List<String>>() {
        @Override/*from  w  ww.j  ava2 s  .  c  om*/
        public List<String> get() {
            if (!isMaster) {
                // First time all nodes start assuming themselves as master and down the line figure out their role
                // Next time onwards, only master will be fetching the list of nodes
                return ImmutableList.of();
            }

            try {
                List<Stats> allNodes;
                List<Stats> failedNodes = ImmutableList.of();

                Request allNodesRequest = prepareGet().setUri(getNodeUri()).build();

                Request failedNodesRequest = prepareGet().setUri(getFailedNodeUri()).build();

                Future allNodesFuture = httpClient.executeAsync(allNodesRequest,
                        createFullJsonResponseHandler(listJsonCodec(Stats.class)));
                Future failedNodesFuture = httpClient.executeAsync(failedNodesRequest,
                        createFullJsonResponseHandler(listJsonCodec(Stats.class)));

                FullJsonResponseHandler.JsonResponse allNodesResponse = (FullJsonResponseHandler.JsonResponse) allNodesFuture
                        .get();
                if (allNodesResponse.getStatusCode() == HttpStatus.OK.code()) {
                    isMaster = true;
                    if (!allNodesResponse.hasValue()) {
                        // Empty result set => server up and only master node running, return localhost has the only node
                        // Do not need to consider failed nodes list as 1node cluster and server is up since it replied to allNodesRequest
                        return ImmutableList.of(InetAddress.getLocalHost().getHostName());
                    } else {
                        allNodes = (List<Stats>) allNodesResponse.getValue();
                    }
                } else {
                    log.info(String.format("v1/node failed with code: %d setting this node as worker ",
                            allNodesResponse.getStatusCode()));
                    isMaster = false;
                    return ImmutableList.of();
                }

                // check on failed nodes
                FullJsonResponseHandler.JsonResponse failedNodesResponse = (FullJsonResponseHandler.JsonResponse) failedNodesFuture
                        .get();
                if (failedNodesResponse.getStatusCode() == HttpStatus.OK.code()) {
                    if (!failedNodesResponse.hasValue()) {
                        failedNodes = ImmutableList.of();
                        //return ImmutableList.of(InetAddress.getLocalHost().getHostName());
                    } else {
                        failedNodes = (List<Stats>) failedNodesResponse.getValue();
                    }
                }

                // keep only the healthy nodes
                allNodes.removeAll(failedNodes);

                Set<String> hosts = new HashSet<String>();

                for (Stats node : allNodes) {
                    hosts.add(node.getUri().getHost());
                    log.debug(String.format("Node: %s", node.getUri()));
                }

                if (hosts.isEmpty()) {
                    // case of master only cluster
                    hosts.add(InetAddress.getLocalHost().getHostName());
                }

                List<String> hostList = Lists.newArrayList(hosts.toArray(new String[0]));
                Collections.sort(hostList);
                return hostList;
            } catch (InterruptedException | ExecutionException | URISyntaxException | UnknownHostException e) {
                throw Throwables.propagate(e);
            }
        }
    }, 10, TimeUnit.SECONDS);
}

From source file:com.qubole.rubix.spi.CacheConfig.java

License:Apache License

public static int getCacheDataExpiration(Configuration conf) {
    return conf.getInt(dataCacheExpirationConf, dataCacheExpiry);
}

From source file:com.qubole.rubix.spi.CacheConfig.java

License:Apache License

public static int getCacheDataExpirationAfterWrite(Configuration conf) {
    return conf.getInt(dataCacheExpirationAfterWriteConf, dataCacheExpiryAfterWrite);
}

From source file:com.qubole.rubix.spi.CacheConfig.java

License:Apache License

public static int getCacheDataFullnessPercentage(Configuration conf) {
    return conf.getInt(dataCacheFullnessConf, dataCacheFullness);
}

From source file:com.qubole.rubix.spi.CacheConfig.java

License:Apache License

public static int getBlockSize(Configuration conf) {
    return conf.getInt(blockSizeConf, blockSize);
}

From source file:com.qubole.rubix.spi.CacheConfig.java

License:Apache License

public static int getServerPort(Configuration conf) {
    return conf.getInt(dataCacheBookkeeperPortConf, serverPort);
}

From source file:com.qubole.rubix.spi.CacheConfig.java

License:Apache License

public static int getServerMaxThreads(Configuration conf) {
    return conf.getInt(dataCacheBookkeeperMaxThreadsConf, serverMaxThreads);
}