Example usage for javax.net.ssl SSLSocket setUseClientMode

List of usage examples for javax.net.ssl SSLSocket setUseClientMode

Introduction

In this page you can find the example usage for javax.net.ssl SSLSocket setUseClientMode.

Prototype

public abstract void setUseClientMode(boolean mode);

Source Link

Document

Configures the socket to use client (or server) mode when handshaking.

Usage

From source file:immf.MyWiser.java

/**
 * Create a new SMTP server with this class as the listener.
 * The default port is 25. Call setPort()/setHostname() before
 * calling start()./*from   ww w  . j  a  v  a  2  s.co m*/
 */
public MyWiser(UsernamePasswordValidator userPass, int port, MyWiserMailListener listener,
        final String tlsKeyStore, final String tlsKeyType, final String tlsKeyPasswd) {
    if (tlsKeyStore == null) {
        log.info("SMTP Server disable TLS");
        this.server = new SMTPServer(this, new EasyAuthenticationHandlerFactory(userPass));
        this.server.setHideTLS(true); // TLS?

    } else {
        // TLS
        log.info("SMTP Server enable TLS");
        this.server = new SMTPServer(this, new EasyAuthenticationHandlerFactory(userPass)) {
            public SSLSocket createSSLSocket(Socket socket) throws IOException {
                SSLSocketFactory sf = createSslSocketFactory(tlsKeyStore, tlsKeyType, tlsKeyPasswd);
                InetSocketAddress remoteAddress = (InetSocketAddress) socket.getRemoteSocketAddress();
                SSLSocket s = (SSLSocket) (sf.createSocket(socket, remoteAddress.getHostName(),
                        socket.getPort(), true));

                s.setUseClientMode(false);

                s.setEnabledCipherSuites(s.getSupportedCipherSuites());

                return s;
            }
        };
        this.server.setRequireTLS(true); // TLS
    }
    this.server.setPort(port);
    this.listener = listener;
}

From source file:com.mendhak.gpslogger.common.network.CertificateValidationWorkflow.java

private void connectToSSLSocket(Socket plainSocket) throws IOException {
    SSLSocketFactory factory = Networks.getSocketFactory(context);
    SSLSocket socket = (SSLSocket) factory.createSocket(host, port);
    if (plainSocket != null) {
        socket = (SSLSocket) factory.createSocket(plainSocket, host, port, true);
    }//  w w w  .  j a  va 2 s.  c  om

    if (serverType == ServerType.SMTP) {
        socket.setUseClientMode(true);
        socket.setNeedClientAuth(true);
    }

    socket.setSoTimeout(5000);
    LOG.debug("Starting handshake...");
    socket.startHandshake();
    SSLSession session = socket.getSession();
    Certificate[] servercerts = session.getPeerCertificates();

}

From source file:it.anyplace.sync.core.security.KeystoreHandler.java

public Socket wrapSocket(Socket socket, boolean isServerSocket, final String... protocols)
        throws KeyManagementException, NoSuchAlgorithmException, KeyStoreException, UnrecoverableKeyException,
        IOException {/*w w  w.j  a v  a 2 s . com*/
    logger.debug("wrapping plain socket, server mode = {}", isServerSocket);
    SSLSocket sslSocket = (SSLSocket) getSocketFactory().createSocket(socket, null, socket.getPort(), true);
    if (isServerSocket) {
        sslSocket.setUseClientMode(false);
    }
    enableALPN(sslSocket, protocols);
    return sslSocket;
}

From source file:ch.cyberduck.core.ftp.FTPClient.java

@Override
protected void sslNegotiation() throws IOException {
    if (protocol.isSecure()) {
        final SSLSocket socket = (SSLSocket) sslSocketFactory.createSocket(_socket_,
                _socket_.getInetAddress().getHostAddress(), _socket_.getPort(), false);
        socket.setEnableSessionCreation(true);
        socket.setUseClientMode(true);
        socket.startHandshake();/*from   www.  j  a v a  2s.c  o  m*/
        _socket_ = socket;
        _controlInput_ = new BufferedReader(
                new InputStreamReader(socket.getInputStream(), getControlEncoding()));
        _controlOutput_ = new BufferedWriter(
                new OutputStreamWriter(socket.getOutputStream(), getControlEncoding()));
    }
}

From source file:com.digitalpebble.storm.crawler.protocol.http.HttpResponse.java

/**
 * Default public constructor.//from   w w  w.ja v a 2  s  .c o  m
 * 
 * @param http
 * @param url
 * @param knownMetadata
 * @throws IOException
 * @throws HttpException
 */
public HttpResponse(HttpProtocol http, URL url, Metadata knownMetadata) throws IOException, HttpException {

    this.http = http;
    this.url = url;

    Scheme scheme = null;

    if ("http".equals(url.getProtocol())) {
        scheme = Scheme.HTTP;
    } else if ("https".equals(url.getProtocol())) {
        scheme = Scheme.HTTPS;
    } else {
        throw new IOException("Unknown scheme (not http/https) for url:" + url);
    }

    String path = "".equals(url.getFile()) ? "/" : url.getFile();

    // some servers will redirect a request with a host line like
    // "Host: <hostname>:80" to "http://<hpstname>/<orig_path>"- they
    // don't want the :80...

    String host = url.getHost();
    int port;
    String portString;
    if (url.getPort() == -1) {
        if (scheme == Scheme.HTTP) {
            port = 80;
        } else {
            port = 443;
        }
        portString = "";
    } else {
        port = url.getPort();
        portString = ":" + port;
    }
    Socket socket = null;

    try {
        socket = new Socket(); // create the socket
        socket.setSoTimeout(http.getTimeout());

        // connect
        String sockHost = http.useProxy() ? http.getProxyHost() : host;
        int sockPort = http.useProxy() ? http.getProxyPort() : port;
        InetSocketAddress sockAddr = new InetSocketAddress(sockHost, sockPort);
        socket.connect(sockAddr, http.getTimeout());

        if (scheme == Scheme.HTTPS) {
            SSLSocketFactory factory = (SSLSocketFactory) SSLSocketFactory.getDefault();
            SSLSocket sslsocket = (SSLSocket) factory.createSocket(socket, sockHost, sockPort, true);
            sslsocket.setUseClientMode(true);

            // Get the protocols and ciphers supported by this JVM
            Set<String> protocols = new HashSet<String>(Arrays.asList(sslsocket.getSupportedProtocols()));
            Set<String> ciphers = new HashSet<String>(Arrays.asList(sslsocket.getSupportedCipherSuites()));

            // Intersect with preferred protocols and ciphers
            protocols.retainAll(http.getTlsPreferredProtocols());
            ciphers.retainAll(http.getTlsPreferredCipherSuites());

            sslsocket.setEnabledProtocols(protocols.toArray(new String[protocols.size()]));
            sslsocket.setEnabledCipherSuites(ciphers.toArray(new String[ciphers.size()]));

            sslsocket.startHandshake();
            socket = sslsocket;
        }

        this.conf = http.getConf();
        if (ConfUtils.getBoolean(conf, "store.ip.address", false) == true) {
            headers.setValue("_ip_", sockAddr.getAddress().getHostAddress());
        }

        // make request
        OutputStream req = socket.getOutputStream();

        StringBuffer reqStr = new StringBuffer("GET ");
        if (http.useProxy()) {
            reqStr.append(url.getProtocol() + "://" + host + portString + path);
        } else {
            reqStr.append(path);
        }

        reqStr.append(" HTTP/1.0\r\n");

        reqStr.append("Host: ");
        reqStr.append(host);
        reqStr.append(portString);
        reqStr.append("\r\n");

        reqStr.append("Accept-Encoding: x-gzip, gzip, deflate\r\n");

        String userAgent = http.getUserAgent();
        if ((userAgent == null) || (userAgent.length() == 0)) {
            if (HttpProtocol.LOGGER.isErrorEnabled()) {
                HttpProtocol.LOGGER.error("User-agent is not set!");
            }
        } else {
            reqStr.append("User-Agent: ");
            reqStr.append(userAgent);
            reqStr.append("\r\n");
        }

        reqStr.append("Accept-Language: ");
        reqStr.append(this.http.getAcceptLanguage());
        reqStr.append("\r\n");

        reqStr.append("Accept: ");
        reqStr.append(this.http.getAccept());
        reqStr.append("\r\n");

        if (knownMetadata != null) {
            String ifModifiedSince = knownMetadata.getFirstValue("cachedLastModified");
            if (StringUtils.isNotBlank(ifModifiedSince)) {
                reqStr.append("If-Modified-Since: ");
                reqStr.append(ifModifiedSince);
                reqStr.append("\r\n");
            }

            String ifNoneMatch = knownMetadata.getFirstValue("cachedEtag");
            if (StringUtils.isNotBlank(ifNoneMatch)) {
                reqStr.append("If-None-Match: ");
                reqStr.append(ifNoneMatch);
                reqStr.append("\r\n");
            }
        }

        reqStr.append("\r\n");

        // @see http://www.w3.org/Protocols/rfc2068/rfc2068.txt for default
        // charset
        // TODO use UTF-8 and set a charset value explicitely
        byte[] reqBytes = reqStr.toString().getBytes(StandardCharsets.ISO_8859_1);

        req.write(reqBytes);
        req.flush();

        PushbackInputStream in = // process response
                new PushbackInputStream(
                        new BufferedInputStream(socket.getInputStream(), HttpProtocol.BUFFER_SIZE),
                        HttpProtocol.BUFFER_SIZE);

        StringBuffer line = new StringBuffer();

        boolean haveSeenNonContinueStatus = false;
        while (!haveSeenNonContinueStatus) {
            // parse status code line
            this.code = parseStatusLine(in, line);
            // parse headers
            parseHeaders(in, line);
            haveSeenNonContinueStatus = code != 100; // 100 is
                                                     // "Continue"
        }
        String transferEncoding = getHeader(HttpHeaders.TRANSFER_ENCODING);
        if (transferEncoding != null && "chunked".equalsIgnoreCase(transferEncoding.trim())) {
            readChunkedContent(in, line);
        } else {
            readPlainContent(in);
        }

        String contentEncoding = getHeader(HttpHeaders.CONTENT_ENCODING);
        if ("gzip".equals(contentEncoding) || "x-gzip".equals(contentEncoding)) {
            content = http.processGzipEncoded(content, url);
        } else if ("deflate".equals(contentEncoding)) {
            content = http.processDeflateEncoded(content, url);
        } else {
            HttpProtocol.LOGGER.trace("fetched {}  bytes from {}", content.length, url);
        }

    } finally {
        if (socket != null)
            socket.close();
    }

}

From source file:org.parosproxy.paros.network.SSLConnector.java

/**
 * Create a SSLsocket using an existing connected socket. It can be used
 * such as a tunneled SSL proxy socket (eg when a CONNECT request is
 * received). This SSLSocket will start server side handshake immediately.
 * /*from www  .ja v a  2 s  .  c  o  m*/
 * @param targethost the host where you want to connect to 
 * @param socket
 * @return
 * @throws IOException
 */
public Socket createTunnelServerSocket(String targethost, Socket socket) throws IOException {
    // ZAP: added host name parameter
    SSLSocket s = (SSLSocket) getTunnelSSLSocketFactory(targethost).createSocket(socket,
            socket.getInetAddress().getHostAddress(), socket.getPort(), true);

    s.setUseClientMode(false);
    s.startHandshake();
    return s;
}

From source file:org.apache.jmeter.protocol.amf.proxy.AmfProxy.java

/**
 * Negotiate a SSL connection.//  w  w w  . j a  v  a  2s  . com
 *
 * @param sock socket in
 * @param host
 * @return a new client socket over ssl
 * @throws Exception if negotiation failed
 */
private Socket startSSL(Socket sock, String host) throws IOException {
    SSLSocketFactory sslFactory = getSSLSocketFactory(host);
    SSLSocket secureSocket;
    if (sslFactory != null) {
        try {
            secureSocket = (SSLSocket) sslFactory.createSocket(sock, sock.getInetAddress().getHostName(),
                    sock.getPort(), true);
            secureSocket.setUseClientMode(false);
            if (log.isDebugEnabled()) {
                log.debug("SSL transaction ok with cipher: " + secureSocket.getSession().getCipherSuite());
            }
            return secureSocket;
        } catch (IOException e) {
            log.error("Error in SSL socket negotiation: ", e);
            throw e;
        }
    } else {
        log.warn("Unable to negotiate SSL transaction, no keystore?");
        throw new IOException("Unable to negotiate SSL transaction, no keystore?");
    }
}

From source file:org.apache.nutch.protocol.http.HttpResponse.java

/**
 * Default public constructor./*  w w w  .  j  a  va2 s  .  c  o m*/
 *
 * @param http
 * @param url
 * @param datum
 * @throws ProtocolException
 * @throws IOException
 */
public HttpResponse(HttpBase http, URL url, CrawlDatum datum) throws ProtocolException, IOException {

    this.http = http;
    this.url = url;
    this.orig = url.toString();
    this.base = url.toString();

    Scheme scheme = null;

    if ("http".equals(url.getProtocol())) {
        scheme = Scheme.HTTP;
    } else if ("https".equals(url.getProtocol())) {
        scheme = Scheme.HTTPS;
    } else {
        throw new HttpException("Unknown scheme (not http/https) for url:" + url);
    }

    if (Http.LOG.isTraceEnabled()) {
        Http.LOG.trace("fetching " + url);
    }

    String path = "".equals(url.getFile()) ? "/" : url.getFile();

    // some servers will redirect a request with a host line like
    // "Host: <hostname>:80" to "http://<hpstname>/<orig_path>"- they
    // don't want the :80...

    LOG.info("Fetching " + url.toString());

    String host = url.getHost();
    int port;
    String portString;
    if (url.getPort() == -1) {
        if (scheme == Scheme.HTTP) {
            port = 80;
        } else {
            port = 443;
        }
        portString = "";
    } else {
        port = url.getPort();
        portString = ":" + port;
    }
    Socket socket = null;

    try {
        socket = new Socket(); // create the socket
        socket.setSoTimeout(http.getTimeout());

        // connect
        String sockHost = http.useProxy(url) ? http.getProxyHost() : host;
        int sockPort = http.useProxy(url) ? http.getProxyPort() : port;
        InetSocketAddress sockAddr = new InetSocketAddress(sockHost, sockPort);
        socket.connect(sockAddr, http.getTimeout());

        if (scheme == Scheme.HTTPS) {
            SSLSocketFactory factory = (SSLSocketFactory) SSLSocketFactory.getDefault();
            SSLSocket sslsocket = (SSLSocket) factory.createSocket(socket, sockHost, sockPort, true);
            sslsocket.setUseClientMode(true);

            // Get the protocols and ciphers supported by this JVM
            Set<String> protocols = new HashSet<String>(Arrays.asList(sslsocket.getSupportedProtocols()));
            Set<String> ciphers = new HashSet<String>(Arrays.asList(sslsocket.getSupportedCipherSuites()));

            // Intersect with preferred protocols and ciphers
            protocols.retainAll(http.getTlsPreferredProtocols());
            ciphers.retainAll(http.getTlsPreferredCipherSuites());

            sslsocket.setEnabledProtocols(protocols.toArray(new String[protocols.size()]));
            sslsocket.setEnabledCipherSuites(ciphers.toArray(new String[ciphers.size()]));

            sslsocket.startHandshake();
            socket = sslsocket;
        }

        this.conf = http.getConf();
        if (sockAddr != null && conf.getBoolean("store.ip.address", false) == true) {
            headers.add("_ip_", sockAddr.getAddress().getHostAddress());
        }

        // make request
        OutputStream req = socket.getOutputStream();

        StringBuffer reqStr = new StringBuffer("GET ");
        if (http.useProxy(url)) {
            reqStr.append(url.getProtocol() + "://" + host + portString + path);
        } else {
            reqStr.append(path);
        }

        reqStr.append(" HTTP/1.0\r\n");

        reqStr.append("Host: ");
        reqStr.append(host);
        reqStr.append(portString);
        reqStr.append("\r\n");

        reqStr.append("Accept-Encoding: x-gzip, gzip, deflate\r\n");

        String userAgent = http.getUserAgent();
        if ((userAgent == null) || (userAgent.length() == 0)) {
            if (Http.LOG.isErrorEnabled()) {
                Http.LOG.error("User-agent is not set!");
            }
        } else {
            reqStr.append("User-Agent: ");
            reqStr.append(userAgent);
            reqStr.append("\r\n");
        }

        reqStr.append("Accept-Language: ");
        reqStr.append(this.http.getAcceptLanguage());
        reqStr.append("\r\n");

        reqStr.append("Accept: ");
        reqStr.append(this.http.getAccept());
        reqStr.append("\r\n");

        if (http.isIfModifiedSinceEnabled() && datum.getModifiedTime() > 0) {
            reqStr.append("If-Modified-Since: " + HttpDateFormat.toString(datum.getModifiedTime()));
            reqStr.append("\r\n");
        }
        reqStr.append("\r\n");

        // store the request in the metadata?
        if (conf.getBoolean("store.http.request", false) == true) {
            headers.add("_request_", reqStr.toString());
        }

        byte[] reqBytes = reqStr.toString().getBytes();

        req.write(reqBytes);
        req.flush();

        LOG.info("Processing response..");

        PushbackInputStream in = // process response
                new PushbackInputStream(new BufferedInputStream(socket.getInputStream(), Http.BUFFER_SIZE),
                        Http.BUFFER_SIZE);

        StringBuffer line = new StringBuffer();

        // store the http headers verbatim
        if (conf.getBoolean("store.http.headers", false) == true) {
            httpHeaders = new StringBuffer();
        }

        headers.add("nutch.fetch.time", Long.toString(System.currentTimeMillis()));

        boolean haveSeenNonContinueStatus = false;
        while (!haveSeenNonContinueStatus) {
            // parse status code line
            this.code = parseStatusLine(in, line);
            if (httpHeaders != null)
                httpHeaders.append(line).append("\n");
            // parse headers
            parseHeaders(in, line, httpHeaders);
            haveSeenNonContinueStatus = code != 100; // 100 is "Continue"
        }

        if (httpHeaders != null) {
            headers.add("_response.headers_", httpHeaders.toString());
        }

        String transferEncoding = getHeader(Response.TRANSFER_ENCODING);
        LOG.info("Transfer Encoding for " + url + ":" + transferEncoding);
        if (transferEncoding != null && "chunked".equalsIgnoreCase(transferEncoding.trim())) {
            readChunkedContent(in, line);
        } else {
            readPlainContent(in);
        }

        String contentEncoding = getHeader(Response.CONTENT_ENCODING);
        if ("gzip".equals(contentEncoding) || "x-gzip".equals(contentEncoding)) {
            content = http.processGzipEncoded(content, url);
        } else if ("deflate".equals(contentEncoding)) {
            content = http.processDeflateEncoded(content, url);
        } else {
            if (Http.LOG.isTraceEnabled()) {
                Http.LOG.trace("fetched " + content.length + " bytes from " + url);
            }
        }

        LOG.info("Checking URL:" + url.toString());
        //check if url contains google drive string
        if (url.toString().toLowerCase().contains("https://drive.google.com/")) {
            //split into two string separated by '=' to get the article id
            LOG.info("Google Drive URL Detected!");
            String[] parts = url.toString().split("=");
            url = new URL("http://drive.google.com/uc?export=download&id=" + parts[1]);

            LOG.info("New URL:" + url.toString());
            this.http = http;
            this.url = url;
            this.orig = url.toString();
            this.base = url.toString();

            HttpClient client = new HttpClient();
            GetMethod method = new GetMethod(url.toString());
            int statusCode = client.executeMethod(method);
            content = method.getResponseBody();
            LOG.info("File Size on Drive: " + content.length);
            //   return;

        }

        LOG.info("Fetch Bytes: " + content.length + " bytes from " + url);

    } finally {
        if (socket != null)
            socket.close();
    }

}

From source file:com.isecpartners.gizmo.HttpRequest.java

private SSLSocket negotiateSSL(Socket sock, String hostname) throws KeyManagementException, KeyStoreException,
        NoSuchAlgorithmException, CertificateException, UnrecoverableKeyException, IOException,
        InvalidKeyException, SignatureException, NoSuchProviderException, NoCertException {
    synchronized (_factories) {
        SSLSocketFactory factory = _factories.get(hostname);

        if (factory == null) {
            factory = initSSL(hostname);
        }//  w  w w  . jav  a 2s .c  o m
        String inbound_hostname = sock.getInetAddress().getHostName();
        int inbound_port = sock.getPort();
        SSLSocket sslsock = (SSLSocket) factory.createSocket(sock, inbound_hostname, inbound_port, true);
        sslsock.setUseClientMode(false);
        if (!sslsock.isConnected()) {
            Logger.getLogger(HttpRequest.class.getName()).log(Level.SEVERE, "Couldn't negotiate SSL");
            System.exit(-1);
        }
        return sslsock;
    }
}

From source file:org.apache.nutch.protocol.s2jh.HttpResponse.java

public HttpResponse(HttpBase http, URL url, WebPage page) throws ProtocolException, IOException {
    conf = http.getConf();//ww w .java2  s .  c o  m
    this.http = http;
    this.url = url;
    Scheme scheme = null;

    if ("http".equals(url.getProtocol())) {
        scheme = Scheme.HTTP;
    } else if ("https".equals(url.getProtocol())) {
        scheme = Scheme.HTTPS;
    } else {
        throw new HttpException("Unknown scheme (not http/https) for url:" + url);
    }

    if (Http.LOG.isTraceEnabled()) {
        Http.LOG.trace("fetching " + url);
    }

    String path = "".equals(url.getFile()) ? "/" : url.getFile();

    // some servers will redirect a request with a host line like
    // "Host: <hostname>:80" to "http://<hpstname>/<orig_path>"- they
    // don't want the :80...

    String host = url.getHost();
    int port;
    String portString;
    if (url.getPort() == -1) {
        if (scheme == Scheme.HTTP) {
            port = 80;
        } else {
            port = 443;
        }
        portString = "";
    } else {
        port = url.getPort();
        portString = ":" + port;
    }
    Socket socket = null;

    try {
        socket = new Socket(); // create the socket
        socket.setSoTimeout(http.getTimeout());

        // connect
        String sockHost = http.useProxy() ? http.getProxyHost() : host;
        int sockPort = http.useProxy() ? http.getProxyPort() : port;
        InetSocketAddress sockAddr = new InetSocketAddress(sockHost, sockPort);
        socket.connect(sockAddr, http.getTimeout());

        if (scheme == Scheme.HTTPS) {
            SSLSocketFactory factory = (SSLSocketFactory) SSLSocketFactory.getDefault();
            SSLSocket sslsocket = (SSLSocket) factory.createSocket(socket, sockHost, sockPort, true);
            sslsocket.setUseClientMode(true);

            // Get the protocols and ciphers supported by this JVM
            Set<String> protocols = new HashSet<String>(Arrays.asList(sslsocket.getSupportedProtocols()));
            Set<String> ciphers = new HashSet<String>(Arrays.asList(sslsocket.getSupportedCipherSuites()));

            // Intersect with preferred protocols and ciphers
            protocols.retainAll(http.getTlsPreferredProtocols());
            ciphers.retainAll(http.getTlsPreferredCipherSuites());

            sslsocket.setEnabledProtocols(protocols.toArray(new String[protocols.size()]));
            sslsocket.setEnabledCipherSuites(ciphers.toArray(new String[ciphers.size()]));

            sslsocket.startHandshake();
            socket = sslsocket;
        }

        if (sockAddr != null && conf.getBoolean("store.ip.address", false) == true) {
            String ipString = sockAddr.getAddress().getHostAddress(); // get the ip
                                                                      // address
            page.getMetadata().put(new Utf8("_ip_"), ByteBuffer.wrap(ipString.getBytes()));
        }

        Http.LOG.debug("HTTP fetching: " + url);
        // make request
        OutputStream req = socket.getOutputStream();

        StringBuffer reqStr = new StringBuffer("GET ");
        if (http.useProxy()) {
            reqStr.append(url.getProtocol() + "://" + host + portString + path);
        } else {
            reqStr.append(path);
        }

        reqStr.append(" HTTP/1.0\r\n");

        reqStr.append("Host: ");
        reqStr.append(host);
        reqStr.append(portString);
        reqStr.append("\r\n");

        reqStr.append("Accept-Encoding: x-gzip, gzip\r\n");

        reqStr.append("Accept: ");
        reqStr.append(this.http.getAccept());
        reqStr.append("\r\n");

        String userAgent = http.getUserAgent();
        if ((userAgent == null) || (userAgent.length() == 0)) {
            if (Http.LOG.isErrorEnabled()) {
                Http.LOG.error("User-agent is not set!");
            }
        } else {
            reqStr.append("User-Agent: ");
            reqStr.append(userAgent);
            reqStr.append("\r\n");
        }

        // if (page.isReadable(WebPage.Field.MODIFIED_TIME.getIndex())) {
        reqStr.append("If-Modified-Since: " + HttpDateFormat.toString(page.getModifiedTime()));
        reqStr.append("\r\n");
        // }
        reqStr.append("\r\n");

        byte[] reqBytes = reqStr.toString().getBytes();

        req.write(reqBytes);
        req.flush();

        PushbackInputStream in = // process response
                new PushbackInputStream(new BufferedInputStream(socket.getInputStream(), Http.BUFFER_SIZE),
                        Http.BUFFER_SIZE);

        StringBuffer line = new StringBuffer();

        boolean haveSeenNonContinueStatus = false;
        while (!haveSeenNonContinueStatus) {
            // parse status code line
            this.code = parseStatusLine(in, line);
            // parse headers
            parseHeaders(in, line);
            haveSeenNonContinueStatus = code != 100; // 100 is "Continue"
        }

        if (!url.toString().endsWith("robots.txt")) {
            if (readPlainContent(url.toString(), in)) {
            } else if (readPlainContentByHtmlunit(url)) {
            } else {
                readPlainContentByWebDriver(url);
            }
        }

        if (content != null && content.length > 0) {
            String html = charset == null ? new String(content) : new String(content, charset);
            //System.out.println("URL: " + url + ", CharsetName: " + charset + " , Page HTML=\n" + html);
            Http.LOG_HTML.trace("URL: " + url + ", CharsetName: " + charset + " , Page HTML=\n" + html);
        }

        // add headers in metadata to row
        if (page.getHeaders() != null) {
            page.getHeaders().clear();
        }
        for (String key : headers.names()) {
            page.getHeaders().put(new Utf8(key), new Utf8(headers.get(key)));
        }

    } catch (Exception e) {
        Http.LOG.error(e.getMessage(), e);
    } finally {
        if (socket != null)
            socket.close();
    }

}