Example usage for java.net InetSocketAddress getAddress

List of usage examples for java.net InetSocketAddress getAddress

  1. HOME
  2. Java
  3. java
  4. java.net.*
  5. InetSocketAddress
  6. getAddress

Introduction

In this page you can find the example usage for java.net InetSocketAddress getAddress.

Prototype

public final InetAddress getAddress()

Source Link

Document

Gets the InetAddress .

Usage

From source file:org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager.java

DatanodeManager(final BlockManager blockManager, final Namesystem namesystem, final Configuration conf)
        throws IOException {
    this.namesystem = namesystem;
    this.blockManager = blockManager;

    this.networktopology = NetworkTopology.getInstance(conf);

    this.heartbeatManager = new HeartbeatManager(namesystem, blockManager, conf, storageMap);
    this.decomManager = new DecommissionManager(namesystem, blockManager, heartbeatManager);
    this.fsClusterStats = newFSClusterStats();

    this.defaultXferPort = NetUtils.createSocketAddr(
            conf.getTrimmed(DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY, DFSConfigKeys.DFS_DATANODE_ADDRESS_DEFAULT))
            .getPort();/*from  ww w  .ja v  a  2 s .c  om*/
    this.defaultInfoPort = NetUtils
            .createSocketAddr(conf.getTrimmed(DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY,
                    DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_DEFAULT))
            .getPort();
    this.defaultInfoSecurePort = NetUtils
            .createSocketAddr(conf.getTrimmed(DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY,
                    DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_DEFAULT))
            .getPort();
    this.defaultIpcPort = NetUtils.createSocketAddr(conf.getTrimmed(DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY,
            DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_DEFAULT)).getPort();
    try {
        this.hostFileManager.refresh(conf.get(DFSConfigKeys.DFS_HOSTS, ""),
                conf.get(DFSConfigKeys.DFS_HOSTS_EXCLUDE, ""));
    } catch (IOException e) {
        LOG.error("error reading hosts files: ", e);
    }
    this.dnsToSwitchMapping = ReflectionUtils
            .newInstance(conf.getClass(DFSConfigKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
                    ScriptBasedMapping.class, DNSToSwitchMapping.class), conf);

    this.rejectUnresolvedTopologyDN = conf.getBoolean(
            DFSConfigKeys.DFS_REJECT_UNRESOLVED_DN_TOPOLOGY_MAPPING_KEY,
            DFSConfigKeys.DFS_REJECT_UNRESOLVED_DN_TOPOLOGY_MAPPING_DEFAULT);

    // If the dns to switch mapping supports cache, resolve network
    // locations of those hosts in the include list and store the mapping
    // in the cache; so future calls to resolve will be fast.
    if (dnsToSwitchMapping instanceof CachedDNSToSwitchMapping) {
        final ArrayList<String> locations = new ArrayList<String>();
        for (InetSocketAddress addr : hostFileManager.getIncludes()) {
            locations.add(addr.getAddress().getHostAddress());
        }
        dnsToSwitchMapping.resolve(locations);
    }

    final long heartbeatIntervalSeconds = conf.getLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
            DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT);
    final int heartbeatRecheckInterval = conf.getInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY,
            DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_DEFAULT); // 5 minutes
    this.heartbeatExpireInterval = 2 * heartbeatRecheckInterval + 10 * 1000 * heartbeatIntervalSeconds;
    final int blockInvalidateLimit = Math.max(20 * (int) (heartbeatIntervalSeconds),
            DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_DEFAULT);
    this.blockInvalidateLimit = conf.getInt(DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY, blockInvalidateLimit);
    LOG.info(DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY + "=" + this.blockInvalidateLimit);

    this.checkIpHostnameInRegistration = conf.getBoolean(
            DFSConfigKeys.DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_KEY,
            DFSConfigKeys.DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_DEFAULT);
    LOG.info(DFSConfigKeys.DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_KEY + "="
            + checkIpHostnameInRegistration);

    this.avoidStaleDataNodesForRead = conf.getBoolean(
            DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY,
            DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_DEFAULT);
    this.avoidStaleDataNodesForWrite = conf.getBoolean(
            DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_KEY,
            DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_DEFAULT);
    this.staleInterval = getStaleIntervalFromConf(conf, heartbeatExpireInterval);
    this.ratioUseStaleDataNodesForWrite = conf.getFloat(
            DFSConfigKeys.DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_KEY,
            DFSConfigKeys.DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_DEFAULT);
    Preconditions.checkArgument((ratioUseStaleDataNodesForWrite > 0 && ratioUseStaleDataNodesForWrite <= 1.0f),
            DFSConfigKeys.DFS_NAMENODE_USE_STALE_DATANODE_FOR_WRITE_RATIO_KEY + " = '"
                    + ratioUseStaleDataNodesForWrite + "' is invalid. "
                    + "It should be a positive non-zero float value, not greater than 1.0f.");
    this.timeBetweenResendingCachingDirectivesMs = conf.getLong(
            DFSConfigKeys.DFS_NAMENODE_PATH_BASED_CACHE_RETRY_INTERVAL_MS,
            DFSConfigKeys.DFS_NAMENODE_PATH_BASED_CACHE_RETRY_INTERVAL_MS_DEFAULT);
    this.blocksPerPostponedMisreplicatedBlocksRescan = conf.getLong(
            DFSConfigKeys.DFS_NAMENODE_BLOCKS_PER_POSTPONEDBLOCKS_RESCAN_KEY,
            DFSConfigKeys.DFS_NAMENODE_BLOCKS_PER_POSTPONEDBLOCKS_RESCAN_KEY_DEFAULT);
}

From source file:org.apache.hadoop.hbase.master.HMaster.java

/**
 * @return Get remote side's InetAddress
 * @throws UnknownHostException//from   ww  w .  j  a  v  a  2s  .  c  o m
 */
InetAddress getRemoteInetAddress(final int port, final long serverStartCode) throws UnknownHostException {
    // Do it out here in its own little method so can fake an address when
    // mocking up in tests.
    InetAddress ia = RpcServer.getRemoteIp();

    // The call could be from the local regionserver,
    // in which case, there is no remote address.
    if (ia == null && serverStartCode == startcode) {
        InetSocketAddress isa = rpcServices.getSocketAddress();
        if (isa != null && isa.getPort() == port) {
            ia = isa.getAddress();
        }
    }
    return ia;
}

From source file:com.digitalpebble.storm.crawler.protocol.http.HttpResponse.java

/**
 * Default public constructor./*w w  w.j  a v  a 2 s  . co  m*/
 * 
 * @param http
 * @param url
 * @param knownMetadata
 * @throws IOException
 * @throws HttpException
 */
public HttpResponse(HttpProtocol http, URL url, Metadata knownMetadata) throws IOException, HttpException {

    this.http = http;
    this.url = url;

    Scheme scheme = null;

    if ("http".equals(url.getProtocol())) {
        scheme = Scheme.HTTP;
    } else if ("https".equals(url.getProtocol())) {
        scheme = Scheme.HTTPS;
    } else {
        throw new IOException("Unknown scheme (not http/https) for url:" + url);
    }

    String path = "".equals(url.getFile()) ? "/" : url.getFile();

    // some servers will redirect a request with a host line like
    // "Host: <hostname>:80" to "http://<hpstname>/<orig_path>"- they
    // don't want the :80...

    String host = url.getHost();
    int port;
    String portString;
    if (url.getPort() == -1) {
        if (scheme == Scheme.HTTP) {
            port = 80;
        } else {
            port = 443;
        }
        portString = "";
    } else {
        port = url.getPort();
        portString = ":" + port;
    }
    Socket socket = null;

    try {
        socket = new Socket(); // create the socket
        socket.setSoTimeout(http.getTimeout());

        // connect
        String sockHost = http.useProxy() ? http.getProxyHost() : host;
        int sockPort = http.useProxy() ? http.getProxyPort() : port;
        InetSocketAddress sockAddr = new InetSocketAddress(sockHost, sockPort);
        socket.connect(sockAddr, http.getTimeout());

        if (scheme == Scheme.HTTPS) {
            SSLSocketFactory factory = (SSLSocketFactory) SSLSocketFactory.getDefault();
            SSLSocket sslsocket = (SSLSocket) factory.createSocket(socket, sockHost, sockPort, true);
            sslsocket.setUseClientMode(true);

            // Get the protocols and ciphers supported by this JVM
            Set<String> protocols = new HashSet<String>(Arrays.asList(sslsocket.getSupportedProtocols()));
            Set<String> ciphers = new HashSet<String>(Arrays.asList(sslsocket.getSupportedCipherSuites()));

            // Intersect with preferred protocols and ciphers
            protocols.retainAll(http.getTlsPreferredProtocols());
            ciphers.retainAll(http.getTlsPreferredCipherSuites());

            sslsocket.setEnabledProtocols(protocols.toArray(new String[protocols.size()]));
            sslsocket.setEnabledCipherSuites(ciphers.toArray(new String[ciphers.size()]));

            sslsocket.startHandshake();
            socket = sslsocket;
        }

        this.conf = http.getConf();
        if (ConfUtils.getBoolean(conf, "store.ip.address", false) == true) {
            headers.setValue("_ip_", sockAddr.getAddress().getHostAddress());
        }

        // make request
        OutputStream req = socket.getOutputStream();

        StringBuffer reqStr = new StringBuffer("GET ");
        if (http.useProxy()) {
            reqStr.append(url.getProtocol() + "://" + host + portString + path);
        } else {
            reqStr.append(path);
        }

        reqStr.append(" HTTP/1.0\r\n");

        reqStr.append("Host: ");
        reqStr.append(host);
        reqStr.append(portString);
        reqStr.append("\r\n");

        reqStr.append("Accept-Encoding: x-gzip, gzip, deflate\r\n");

        String userAgent = http.getUserAgent();
        if ((userAgent == null) || (userAgent.length() == 0)) {
            if (HttpProtocol.LOGGER.isErrorEnabled()) {
                HttpProtocol.LOGGER.error("User-agent is not set!");
            }
        } else {
            reqStr.append("User-Agent: ");
            reqStr.append(userAgent);
            reqStr.append("\r\n");
        }

        reqStr.append("Accept-Language: ");
        reqStr.append(this.http.getAcceptLanguage());
        reqStr.append("\r\n");

        reqStr.append("Accept: ");
        reqStr.append(this.http.getAccept());
        reqStr.append("\r\n");

        if (knownMetadata != null) {
            String ifModifiedSince = knownMetadata.getFirstValue("cachedLastModified");
            if (StringUtils.isNotBlank(ifModifiedSince)) {
                reqStr.append("If-Modified-Since: ");
                reqStr.append(ifModifiedSince);
                reqStr.append("\r\n");
            }

            String ifNoneMatch = knownMetadata.getFirstValue("cachedEtag");
            if (StringUtils.isNotBlank(ifNoneMatch)) {
                reqStr.append("If-None-Match: ");
                reqStr.append(ifNoneMatch);
                reqStr.append("\r\n");
            }
        }

        reqStr.append("\r\n");

        // @see http://www.w3.org/Protocols/rfc2068/rfc2068.txt for default
        // charset
        // TODO use UTF-8 and set a charset value explicitely
        byte[] reqBytes = reqStr.toString().getBytes(StandardCharsets.ISO_8859_1);

        req.write(reqBytes);
        req.flush();

        PushbackInputStream in = // process response
                new PushbackInputStream(
                        new BufferedInputStream(socket.getInputStream(), HttpProtocol.BUFFER_SIZE),
                        HttpProtocol.BUFFER_SIZE);

        StringBuffer line = new StringBuffer();

        boolean haveSeenNonContinueStatus = false;
        while (!haveSeenNonContinueStatus) {
            // parse status code line
            this.code = parseStatusLine(in, line);
            // parse headers
            parseHeaders(in, line);
            haveSeenNonContinueStatus = code != 100; // 100 is
                                                     // "Continue"
        }
        String transferEncoding = getHeader(HttpHeaders.TRANSFER_ENCODING);
        if (transferEncoding != null && "chunked".equalsIgnoreCase(transferEncoding.trim())) {
            readChunkedContent(in, line);
        } else {
            readPlainContent(in);
        }

        String contentEncoding = getHeader(HttpHeaders.CONTENT_ENCODING);
        if ("gzip".equals(contentEncoding) || "x-gzip".equals(contentEncoding)) {
            content = http.processGzipEncoded(content, url);
        } else if ("deflate".equals(contentEncoding)) {
            content = http.processDeflateEncoded(content, url);
        } else {
            HttpProtocol.LOGGER.trace("fetched {}  bytes from {}", content.length, url);
        }

    } finally {
        if (socket != null)
            socket.close();
    }

}

From source file:org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl.java

private NodeId buildNodeId(InetSocketAddress connectAddress, String hostOverride) {
    if (hostOverride != null) {
        connectAddress = NetUtils//from  w w  w  . j  a v  a2  s  . c o  m
                .getConnectAddress(new InetSocketAddress(hostOverride, connectAddress.getPort()));
    }
    return NodeId.newInstance(connectAddress.getAddress().getCanonicalHostName(), connectAddress.getPort());
}

From source file:org.apache.hadoop.hbase.regionserver.RSRpcServices.java

public RSRpcServices(HRegionServer rs) throws IOException {
    regionServer = rs;/*  w  ww  .  j a v  a 2s  .  c om*/

    RpcSchedulerFactory rpcSchedulerFactory;
    try {
        Class<?> rpcSchedulerFactoryClass = rs.conf.getClass(REGION_SERVER_RPC_SCHEDULER_FACTORY_CLASS,
                SimpleRpcSchedulerFactory.class);
        rpcSchedulerFactory = ((RpcSchedulerFactory) rpcSchedulerFactoryClass.newInstance());
    } catch (InstantiationException e) {
        throw new IllegalArgumentException(e);
    } catch (IllegalAccessException e) {
        throw new IllegalArgumentException(e);
    }
    // Server to handle client requests.
    String hostname = rs.conf.get("hbase.regionserver.ipc.address",
            Strings.domainNamePointerToHostName(
                    DNS.getDefaultHost(rs.conf.get("hbase.regionserver.dns.interface", "default"),
                            rs.conf.get("hbase.regionserver.dns.nameserver", "default"))));
    int port = rs.conf.getInt(HConstants.REGIONSERVER_PORT, HConstants.DEFAULT_REGIONSERVER_PORT);
    // Creation of a HSA will force a resolve.
    InetSocketAddress initialIsa = new InetSocketAddress(hostname, port);
    if (initialIsa.getAddress() == null) {
        throw new IllegalArgumentException("Failed resolve of " + initialIsa);
    }
    priority = new AnnotationReadingPriorityFunction(this);
    String name = rs.getProcessName() + "/" + initialIsa.toString();
    // Set how many times to retry talking to another server over HConnection.
    ConnectionUtils.setServerSideHConnectionRetriesConfig(rs.conf, name, LOG);
    rpcServer = new RpcServer(rs, name, getServices(), initialIsa, // BindAddress is IP we got for this server.
            rs.conf, rpcSchedulerFactory.create(rs.conf, this));

    scannerLeaseTimeoutPeriod = rs.conf.getInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD,
            HConstants.DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD);
    maxScannerResultSize = rs.conf.getLong(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,
            HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);

    // Set our address.
    isa = rpcServer.getListenerAddress();
    rpcServer.setErrorHandler(this);
    rs.setName(name);
}

From source file:com.alibaba.wasp.master.FMaster.java

/**
 * Initializes the FMaster. The steps are as follows:
 * <p>/* w w w  .  ja  va2  s . com*/
 * <ol>
 * <li>Initialize FMaster RPC and address
 * <li>Connect to ZooKeeper.
 * </ol>
 * <p>
 * Remaining steps of initialization occur in {@link #run()} so that they run
 * in their own thread rather than within the context of the constructor.
 * 
 * @throws InterruptedException
 */
public FMaster(final Configuration conf) throws IOException, KeeperException, InterruptedException {
    this.conf = new Configuration(conf);
    // Set how many times to retry talking to another server over HConnection.
    FConnectionManager.setServerSideFConnectionRetries(this.conf, LOG);
    // Server to handle client requests.
    String hostname = Strings
            .domainNamePointerToHostName(DNS.getDefaultHost(conf.get("wasp.master.dns.interface", "default"),
                    conf.get("wasp.master.dns.nameserver", "default")));
    int port = conf.getInt(FConstants.MASTER_PORT, FConstants.DEFAULT_MASTER_PORT);
    // Creation of a ISA will force a resolve.
    InetSocketAddress initialIsa = new InetSocketAddress(hostname, port);
    if (initialIsa.getAddress() == null) {
        throw new IllegalArgumentException("Failed resolve of " + initialIsa);
    }
    this.rpcServer = WaspRPC.getServer(FMaster.class, this,
            new Class<?>[] { FMasterMonitorProtocol.class, FMasterAdminProtocol.class,
                    FServerStatusProtocol.class, FMetaServerProtocol.class },
            initialIsa.getHostName(), // BindAddress is IP we got for this server.
            initialIsa.getPort(), conf);
    // Set our address.
    this.isa = this.rpcServer.getListenerAddress();
    this.serverName = new ServerName(this.isa.getHostName(), this.isa.getPort(), System.currentTimeMillis());

    // set the thread name now we have an address
    setName(MASTER + "-" + this.serverName.toString());

    this.zooKeeper = new ZooKeeperWatcher(conf, MASTER + ":" + isa.getPort(), this, true);

    // metrics interval: using the same property as fserver.
    this.msgInterval = conf.getInt("wasp.fserver.msginterval", 3 * 1000);

    this.metricsMaster = new MetricsMaster(new MetricsMasterWrapperImpl(this));
}

From source file:org.apache.nutch.protocol.s2jh.HttpResponse.java

public HttpResponse(HttpBase http, URL url, WebPage page) throws ProtocolException, IOException {
    conf = http.getConf();//ww w .j  a  va 2  s. c  om
    this.http = http;
    this.url = url;
    Scheme scheme = null;

    if ("http".equals(url.getProtocol())) {
        scheme = Scheme.HTTP;
    } else if ("https".equals(url.getProtocol())) {
        scheme = Scheme.HTTPS;
    } else {
        throw new HttpException("Unknown scheme (not http/https) for url:" + url);
    }

    if (Http.LOG.isTraceEnabled()) {
        Http.LOG.trace("fetching " + url);
    }

    String path = "".equals(url.getFile()) ? "/" : url.getFile();

    // some servers will redirect a request with a host line like
    // "Host: <hostname>:80" to "http://<hpstname>/<orig_path>"- they
    // don't want the :80...

    String host = url.getHost();
    int port;
    String portString;
    if (url.getPort() == -1) {
        if (scheme == Scheme.HTTP) {
            port = 80;
        } else {
            port = 443;
        }
        portString = "";
    } else {
        port = url.getPort();
        portString = ":" + port;
    }
    Socket socket = null;

    try {
        socket = new Socket(); // create the socket
        socket.setSoTimeout(http.getTimeout());

        // connect
        String sockHost = http.useProxy() ? http.getProxyHost() : host;
        int sockPort = http.useProxy() ? http.getProxyPort() : port;
        InetSocketAddress sockAddr = new InetSocketAddress(sockHost, sockPort);
        socket.connect(sockAddr, http.getTimeout());

        if (scheme == Scheme.HTTPS) {
            SSLSocketFactory factory = (SSLSocketFactory) SSLSocketFactory.getDefault();
            SSLSocket sslsocket = (SSLSocket) factory.createSocket(socket, sockHost, sockPort, true);
            sslsocket.setUseClientMode(true);

            // Get the protocols and ciphers supported by this JVM
            Set<String> protocols = new HashSet<String>(Arrays.asList(sslsocket.getSupportedProtocols()));
            Set<String> ciphers = new HashSet<String>(Arrays.asList(sslsocket.getSupportedCipherSuites()));

            // Intersect with preferred protocols and ciphers
            protocols.retainAll(http.getTlsPreferredProtocols());
            ciphers.retainAll(http.getTlsPreferredCipherSuites());

            sslsocket.setEnabledProtocols(protocols.toArray(new String[protocols.size()]));
            sslsocket.setEnabledCipherSuites(ciphers.toArray(new String[ciphers.size()]));

            sslsocket.startHandshake();
            socket = sslsocket;
        }

        if (sockAddr != null && conf.getBoolean("store.ip.address", false) == true) {
            String ipString = sockAddr.getAddress().getHostAddress(); // get the ip
                                                                      // address
            page.getMetadata().put(new Utf8("_ip_"), ByteBuffer.wrap(ipString.getBytes()));
        }

        Http.LOG.debug("HTTP fetching: " + url);
        // make request
        OutputStream req = socket.getOutputStream();

        StringBuffer reqStr = new StringBuffer("GET ");
        if (http.useProxy()) {
            reqStr.append(url.getProtocol() + "://" + host + portString + path);
        } else {
            reqStr.append(path);
        }

        reqStr.append(" HTTP/1.0\r\n");

        reqStr.append("Host: ");
        reqStr.append(host);
        reqStr.append(portString);
        reqStr.append("\r\n");

        reqStr.append("Accept-Encoding: x-gzip, gzip\r\n");

        reqStr.append("Accept: ");
        reqStr.append(this.http.getAccept());
        reqStr.append("\r\n");

        String userAgent = http.getUserAgent();
        if ((userAgent == null) || (userAgent.length() == 0)) {
            if (Http.LOG.isErrorEnabled()) {
                Http.LOG.error("User-agent is not set!");
            }
        } else {
            reqStr.append("User-Agent: ");
            reqStr.append(userAgent);
            reqStr.append("\r\n");
        }

        // if (page.isReadable(WebPage.Field.MODIFIED_TIME.getIndex())) {
        reqStr.append("If-Modified-Since: " + HttpDateFormat.toString(page.getModifiedTime()));
        reqStr.append("\r\n");
        // }
        reqStr.append("\r\n");

        byte[] reqBytes = reqStr.toString().getBytes();

        req.write(reqBytes);
        req.flush();

        PushbackInputStream in = // process response
                new PushbackInputStream(new BufferedInputStream(socket.getInputStream(), Http.BUFFER_SIZE),
                        Http.BUFFER_SIZE);

        StringBuffer line = new StringBuffer();

        boolean haveSeenNonContinueStatus = false;
        while (!haveSeenNonContinueStatus) {
            // parse status code line
            this.code = parseStatusLine(in, line);
            // parse headers
            parseHeaders(in, line);
            haveSeenNonContinueStatus = code != 100; // 100 is "Continue"
        }

        if (!url.toString().endsWith("robots.txt")) {
            if (readPlainContent(url.toString(), in)) {
            } else if (readPlainContentByHtmlunit(url)) {
            } else {
                readPlainContentByWebDriver(url);
            }
        }

        if (content != null && content.length > 0) {
            String html = charset == null ? new String(content) : new String(content, charset);
            //System.out.println("URL: " + url + ", CharsetName: " + charset + " , Page HTML=\n" + html);
            Http.LOG_HTML.trace("URL: " + url + ", CharsetName: " + charset + " , Page HTML=\n" + html);
        }

        // add headers in metadata to row
        if (page.getHeaders() != null) {
            page.getHeaders().clear();
        }
        for (String key : headers.names()) {
            page.getHeaders().put(new Utf8(key), new Utf8(headers.get(key)));
        }

    } catch (Exception e) {
        Http.LOG.error(e.getMessage(), e);
    } finally {
        if (socket != null)
            socket.close();
    }

}

From source file:org.apache.nutch.protocol.http.HttpResponse.java

/**
 * Default public constructor./*  www .j av  a 2 s  . co m*/
 *
 * @param http
 * @param url
 * @param datum
 * @throws ProtocolException
 * @throws IOException
 */
public HttpResponse(HttpBase http, URL url, CrawlDatum datum) throws ProtocolException, IOException {

    this.http = http;
    this.url = url;
    this.orig = url.toString();
    this.base = url.toString();

    Scheme scheme = null;

    if ("http".equals(url.getProtocol())) {
        scheme = Scheme.HTTP;
    } else if ("https".equals(url.getProtocol())) {
        scheme = Scheme.HTTPS;
    } else {
        throw new HttpException("Unknown scheme (not http/https) for url:" + url);
    }

    if (Http.LOG.isTraceEnabled()) {
        Http.LOG.trace("fetching " + url);
    }

    String path = "".equals(url.getFile()) ? "/" : url.getFile();

    // some servers will redirect a request with a host line like
    // "Host: <hostname>:80" to "http://<hpstname>/<orig_path>"- they
    // don't want the :80...

    LOG.info("Fetching " + url.toString());

    String host = url.getHost();
    int port;
    String portString;
    if (url.getPort() == -1) {
        if (scheme == Scheme.HTTP) {
            port = 80;
        } else {
            port = 443;
        }
        portString = "";
    } else {
        port = url.getPort();
        portString = ":" + port;
    }
    Socket socket = null;

    try {
        socket = new Socket(); // create the socket
        socket.setSoTimeout(http.getTimeout());

        // connect
        String sockHost = http.useProxy(url) ? http.getProxyHost() : host;
        int sockPort = http.useProxy(url) ? http.getProxyPort() : port;
        InetSocketAddress sockAddr = new InetSocketAddress(sockHost, sockPort);
        socket.connect(sockAddr, http.getTimeout());

        if (scheme == Scheme.HTTPS) {
            SSLSocketFactory factory = (SSLSocketFactory) SSLSocketFactory.getDefault();
            SSLSocket sslsocket = (SSLSocket) factory.createSocket(socket, sockHost, sockPort, true);
            sslsocket.setUseClientMode(true);

            // Get the protocols and ciphers supported by this JVM
            Set<String> protocols = new HashSet<String>(Arrays.asList(sslsocket.getSupportedProtocols()));
            Set<String> ciphers = new HashSet<String>(Arrays.asList(sslsocket.getSupportedCipherSuites()));

            // Intersect with preferred protocols and ciphers
            protocols.retainAll(http.getTlsPreferredProtocols());
            ciphers.retainAll(http.getTlsPreferredCipherSuites());

            sslsocket.setEnabledProtocols(protocols.toArray(new String[protocols.size()]));
            sslsocket.setEnabledCipherSuites(ciphers.toArray(new String[ciphers.size()]));

            sslsocket.startHandshake();
            socket = sslsocket;
        }

        this.conf = http.getConf();
        if (sockAddr != null && conf.getBoolean("store.ip.address", false) == true) {
            headers.add("_ip_", sockAddr.getAddress().getHostAddress());
        }

        // make request
        OutputStream req = socket.getOutputStream();

        StringBuffer reqStr = new StringBuffer("GET ");
        if (http.useProxy(url)) {
            reqStr.append(url.getProtocol() + "://" + host + portString + path);
        } else {
            reqStr.append(path);
        }

        reqStr.append(" HTTP/1.0\r\n");

        reqStr.append("Host: ");
        reqStr.append(host);
        reqStr.append(portString);
        reqStr.append("\r\n");

        reqStr.append("Accept-Encoding: x-gzip, gzip, deflate\r\n");

        String userAgent = http.getUserAgent();
        if ((userAgent == null) || (userAgent.length() == 0)) {
            if (Http.LOG.isErrorEnabled()) {
                Http.LOG.error("User-agent is not set!");
            }
        } else {
            reqStr.append("User-Agent: ");
            reqStr.append(userAgent);
            reqStr.append("\r\n");
        }

        reqStr.append("Accept-Language: ");
        reqStr.append(this.http.getAcceptLanguage());
        reqStr.append("\r\n");

        reqStr.append("Accept: ");
        reqStr.append(this.http.getAccept());
        reqStr.append("\r\n");

        if (http.isIfModifiedSinceEnabled() && datum.getModifiedTime() > 0) {
            reqStr.append("If-Modified-Since: " + HttpDateFormat.toString(datum.getModifiedTime()));
            reqStr.append("\r\n");
        }
        reqStr.append("\r\n");

        // store the request in the metadata?
        if (conf.getBoolean("store.http.request", false) == true) {
            headers.add("_request_", reqStr.toString());
        }

        byte[] reqBytes = reqStr.toString().getBytes();

        req.write(reqBytes);
        req.flush();

        LOG.info("Processing response..");

        PushbackInputStream in = // process response
                new PushbackInputStream(new BufferedInputStream(socket.getInputStream(), Http.BUFFER_SIZE),
                        Http.BUFFER_SIZE);

        StringBuffer line = new StringBuffer();

        // store the http headers verbatim
        if (conf.getBoolean("store.http.headers", false) == true) {
            httpHeaders = new StringBuffer();
        }

        headers.add("nutch.fetch.time", Long.toString(System.currentTimeMillis()));

        boolean haveSeenNonContinueStatus = false;
        while (!haveSeenNonContinueStatus) {
            // parse status code line
            this.code = parseStatusLine(in, line);
            if (httpHeaders != null)
                httpHeaders.append(line).append("\n");
            // parse headers
            parseHeaders(in, line, httpHeaders);
            haveSeenNonContinueStatus = code != 100; // 100 is "Continue"
        }

        if (httpHeaders != null) {
            headers.add("_response.headers_", httpHeaders.toString());
        }

        String transferEncoding = getHeader(Response.TRANSFER_ENCODING);
        LOG.info("Transfer Encoding for " + url + ":" + transferEncoding);
        if (transferEncoding != null && "chunked".equalsIgnoreCase(transferEncoding.trim())) {
            readChunkedContent(in, line);
        } else {
            readPlainContent(in);
        }

        String contentEncoding = getHeader(Response.CONTENT_ENCODING);
        if ("gzip".equals(contentEncoding) || "x-gzip".equals(contentEncoding)) {
            content = http.processGzipEncoded(content, url);
        } else if ("deflate".equals(contentEncoding)) {
            content = http.processDeflateEncoded(content, url);
        } else {
            if (Http.LOG.isTraceEnabled()) {
                Http.LOG.trace("fetched " + content.length + " bytes from " + url);
            }
        }

        LOG.info("Checking URL:" + url.toString());
        //check if url contains google drive string
        if (url.toString().toLowerCase().contains("https://drive.google.com/")) {
            //split into two string separated by '=' to get the article id
            LOG.info("Google Drive URL Detected!");
            String[] parts = url.toString().split("=");
            url = new URL("http://drive.google.com/uc?export=download&id=" + parts[1]);

            LOG.info("New URL:" + url.toString());
            this.http = http;
            this.url = url;
            this.orig = url.toString();
            this.base = url.toString();

            HttpClient client = new HttpClient();
            GetMethod method = new GetMethod(url.toString());
            int statusCode = client.executeMethod(method);
            content = method.getResponseBody();
            LOG.info("File Size on Drive: " + content.length);
            //   return;

        }

        LOG.info("Fetch Bytes: " + content.length + " bytes from " + url);

    } finally {
        if (socket != null)
            socket.close();
    }

}

From source file:org.apache.hadoop.raid.RaidNode.java

private void startHttpServer() throws IOException {
    String infoAddr = conf.get("mapred.raid.http.address", "localhost:50091");
    InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(infoAddr);
    this.infoBindAddress = infoSocAddr.getAddress().getHostAddress();
    int tmpInfoPort = infoSocAddr.getPort();
    this.infoServer = new HttpServer("raid", this.infoBindAddress, tmpInfoPort, tmpInfoPort == 0, conf);
    this.infoServer.setAttribute("raidnode", this);
    this.infoServer.addInternalServlet("corruptfilecounter", "/corruptfilecounter",
            CorruptFileCounterServlet.class);
    this.infoServer.start();
    LOG.info("Web server started at port " + this.infoServer.getPort());
}

From source file:com.mellanox.r4h.DFSClient.java

public static boolean isLocalAddress(InetSocketAddress targetAddr) {
    InetAddress addr = targetAddr.getAddress();
    Boolean cached = localAddrMap.get(addr.getHostAddress());
    if (cached != null) {
        if (LOG.isTraceEnabled()) {
            LOG.trace("Address " + targetAddr + (cached ? " is local" : " is not local"));
        }//from   w  ww. j ava2  s .c o m
        return cached;
    }

    boolean local = NetUtils.isLocalAddress(addr);

    if (LOG.isTraceEnabled()) {
        LOG.trace("Address " + targetAddr + (local ? " is local" : " is not local"));
    }
    localAddrMap.put(addr.getHostAddress(), local);
    return local;
}