Example usage for java.util Set retainAll

List of usage examples for java.util Set retainAll

  1. HOME
  2. Java
  3. java
  4. java.util.*
  5. Set
  6. retainAll

Introduction

In this page you can find the example usage for java.util Set retainAll.

Prototype

boolean retainAll(Collection<?> c);

Source Link

Document

Retains only the elements in this set that are contained in the specified collection (optional operation).

Usage

From source file:com.datumbox.framework.core.common.dataobjects.Dataframe.java

/**
 * Removes completely a list of columns from the dataset. The meta-data of
 * the Dataframe are updated. The method internally uses threads.
 *
 * @param columnSet/* ww  w.j a v  a2  s.co m*/
 */
public void dropXColumns(Set<Object> columnSet) {
    columnSet.retainAll(data.xDataTypes.keySet()); //keep only those columns that are already known to the Meta data of the Dataframe

    if (columnSet.isEmpty()) {
        return;
    }

    //remove all the columns from the Meta data
    data.xDataTypes.keySet().removeAll(columnSet);

    streamExecutor.forEach(StreamMethods.stream(entries(), true), e -> {
        Integer rId = e.getKey();
        Record r = e.getValue();

        AssociativeArray xData = r.getX().copy();
        boolean modified = xData.keySet().removeAll(columnSet);

        if (modified) {
            Record newR = new Record(xData, r.getY(), r.getYPredicted(), r.getYPredictedProbabilities());

            //safe to call in this context. we already updated the meta when we modified the xDataTypes
            _unsafe_set(rId, newR);
        }
    });

}

From source file:org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.FDsAndEquivClassesVisitor.java

/***
 * Propagated equivalent classes from the child to the current operator,
 * based on the used variables of the current operator.
 *
 * @param op//ww w .  ja  v  a  2  s. c om
 *            , the current operator
 * @param ctx
 *            , the optimization context which keeps track of all equivalent
 *            classes.
 * @param usedVariables
 *            , used variables.
 * @throws AlgebricksException
 */
private void propagateFDsAndEquivClassesForUsedVars(ILogicalOperator op, IOptimizationContext ctx,
        List<LogicalVariable> usedVariables) throws AlgebricksException {
    ILogicalOperator op2 = op.getInputs().get(0).getValue();
    Map<LogicalVariable, EquivalenceClass> eqClasses = getOrCreateEqClasses(op, ctx);
    List<FunctionalDependency> fds = new ArrayList<FunctionalDependency>();
    ctx.putFDList(op, fds);

    Map<LogicalVariable, EquivalenceClass> chldClasses = getOrComputeEqClasses(op2, ctx);

    // Propagate equivalent classes that contain the used variables.
    for (LogicalVariable v : usedVariables) {
        EquivalenceClass ec = eqClasses.get(v);
        if (ec == null) {
            EquivalenceClass oc = chldClasses.get(v);
            if (oc == null) {
                continue;
            }
            List<LogicalVariable> m = new LinkedList<LogicalVariable>();
            for (LogicalVariable v2 : oc.getMembers()) {
                if (usedVariables.contains(v2)) {
                    m.add(v2);
                }
            }
            EquivalenceClass nc;
            if (oc.representativeIsConst()) {
                nc = new EquivalenceClass(m, oc.getConstRepresentative());
            } else if (m.contains(oc.getVariableRepresentative())) {
                nc = new EquivalenceClass(m, oc.getVariableRepresentative());
            } else {
                nc = new EquivalenceClass(m, v);
            }
            for (LogicalVariable v3 : m) {
                eqClasses.put(v3, nc);
            }
        }
    }

    // Propagates equivalent classes that contain expressions that use the
    // used variables.
    // Note that for the case variable $v is not in the used variables but
    // it is
    // equivalent to field-access($t, i) and $t is a used variable, the
    // equivalent
    // class should still be propagated (kept).
    Set<LogicalVariable> usedVarSet = new HashSet<LogicalVariable>(usedVariables);
    for (Entry<LogicalVariable, EquivalenceClass> entry : chldClasses.entrySet()) {
        EquivalenceClass ec = entry.getValue();
        for (ILogicalExpression expr : ec.getExpressionMembers()) {
            Set<LogicalVariable> exprUsedVars = new HashSet<LogicalVariable>();
            expr.getUsedVariables(exprUsedVars);
            exprUsedVars.retainAll(usedVarSet);
            // Check if the expression member uses a used variable.
            if (!exprUsedVars.isEmpty()) {
                for (LogicalVariable v : ec.getMembers()) {
                    eqClasses.put(v, ec);
                    // If variable members contain a used variable, the
                    // representative
                    // variable should be a used variable.
                    if (usedVarSet.contains(v)) {
                        ec.setVariableRepresentative(v);
                    }
                }
            }
        }
    }

    List<FunctionalDependency> chldFds = getOrComputeFDs(op2, ctx);
    for (FunctionalDependency fd : chldFds) {
        if (!usedVariables.containsAll(fd.getHead())) {
            continue;
        }
        List<LogicalVariable> tl = new LinkedList<LogicalVariable>();
        for (LogicalVariable v : fd.getTail()) {
            if (usedVariables.contains(v)) {
                tl.add(v);
            }
        }
        if (!tl.isEmpty()) {
            FunctionalDependency newFd = new FunctionalDependency(fd.getHead(), tl);
            fds.add(newFd);
        }
    }
}

From source file:org.sigmah.client.util.ClientUtils.java

/**
 * Utility method returning the intersection between the given sets (i.e. elements present among both sets).
 * /*ww  w  .  j ava2 s. c om*/
 * @param firstSet
 *          The first set.
 * @param secondSet
 *          The second set.
 * @return The intersection between the given sets (i.e. elements present among both sets).<br/>
 *         Never returns {@code null} ; if no intersection, returns empty set.
 */
public static <T> Set<T> intersect(final Set<T> firstSet, final Set<T> secondSet) {

    if (isEmpty(firstSet)) {
        return secondSet == null ? new HashSet<T>(0) : secondSet;
    }
    if (isEmpty(secondSet)) {
        return firstSet;
    }

    firstSet.retainAll(secondSet);

    return firstSet;
}

From source file:org.apache.nutch.protocol.http.HttpResponse.java

/**
 * Default public constructor./*from   w w  w. j  a v a 2 s.co m*/
 *
 * @param http
 * @param url
 * @param datum
 * @throws ProtocolException
 * @throws IOException
 */
public HttpResponse(HttpBase http, URL url, CrawlDatum datum) throws ProtocolException, IOException {

    this.http = http;
    this.url = url;
    this.orig = url.toString();
    this.base = url.toString();

    Scheme scheme = null;

    if ("http".equals(url.getProtocol())) {
        scheme = Scheme.HTTP;
    } else if ("https".equals(url.getProtocol())) {
        scheme = Scheme.HTTPS;
    } else {
        throw new HttpException("Unknown scheme (not http/https) for url:" + url);
    }

    if (Http.LOG.isTraceEnabled()) {
        Http.LOG.trace("fetching " + url);
    }

    String path = "".equals(url.getFile()) ? "/" : url.getFile();

    // some servers will redirect a request with a host line like
    // "Host: <hostname>:80" to "http://<hpstname>/<orig_path>"- they
    // don't want the :80...

    LOG.info("Fetching " + url.toString());

    String host = url.getHost();
    int port;
    String portString;
    if (url.getPort() == -1) {
        if (scheme == Scheme.HTTP) {
            port = 80;
        } else {
            port = 443;
        }
        portString = "";
    } else {
        port = url.getPort();
        portString = ":" + port;
    }
    Socket socket = null;

    try {
        socket = new Socket(); // create the socket
        socket.setSoTimeout(http.getTimeout());

        // connect
        String sockHost = http.useProxy(url) ? http.getProxyHost() : host;
        int sockPort = http.useProxy(url) ? http.getProxyPort() : port;
        InetSocketAddress sockAddr = new InetSocketAddress(sockHost, sockPort);
        socket.connect(sockAddr, http.getTimeout());

        if (scheme == Scheme.HTTPS) {
            SSLSocketFactory factory = (SSLSocketFactory) SSLSocketFactory.getDefault();
            SSLSocket sslsocket = (SSLSocket) factory.createSocket(socket, sockHost, sockPort, true);
            sslsocket.setUseClientMode(true);

            // Get the protocols and ciphers supported by this JVM
            Set<String> protocols = new HashSet<String>(Arrays.asList(sslsocket.getSupportedProtocols()));
            Set<String> ciphers = new HashSet<String>(Arrays.asList(sslsocket.getSupportedCipherSuites()));

            // Intersect with preferred protocols and ciphers
            protocols.retainAll(http.getTlsPreferredProtocols());
            ciphers.retainAll(http.getTlsPreferredCipherSuites());

            sslsocket.setEnabledProtocols(protocols.toArray(new String[protocols.size()]));
            sslsocket.setEnabledCipherSuites(ciphers.toArray(new String[ciphers.size()]));

            sslsocket.startHandshake();
            socket = sslsocket;
        }

        this.conf = http.getConf();
        if (sockAddr != null && conf.getBoolean("store.ip.address", false) == true) {
            headers.add("_ip_", sockAddr.getAddress().getHostAddress());
        }

        // make request
        OutputStream req = socket.getOutputStream();

        StringBuffer reqStr = new StringBuffer("GET ");
        if (http.useProxy(url)) {
            reqStr.append(url.getProtocol() + "://" + host + portString + path);
        } else {
            reqStr.append(path);
        }

        reqStr.append(" HTTP/1.0\r\n");

        reqStr.append("Host: ");
        reqStr.append(host);
        reqStr.append(portString);
        reqStr.append("\r\n");

        reqStr.append("Accept-Encoding: x-gzip, gzip, deflate\r\n");

        String userAgent = http.getUserAgent();
        if ((userAgent == null) || (userAgent.length() == 0)) {
            if (Http.LOG.isErrorEnabled()) {
                Http.LOG.error("User-agent is not set!");
            }
        } else {
            reqStr.append("User-Agent: ");
            reqStr.append(userAgent);
            reqStr.append("\r\n");
        }

        reqStr.append("Accept-Language: ");
        reqStr.append(this.http.getAcceptLanguage());
        reqStr.append("\r\n");

        reqStr.append("Accept: ");
        reqStr.append(this.http.getAccept());
        reqStr.append("\r\n");

        if (http.isIfModifiedSinceEnabled() && datum.getModifiedTime() > 0) {
            reqStr.append("If-Modified-Since: " + HttpDateFormat.toString(datum.getModifiedTime()));
            reqStr.append("\r\n");
        }
        reqStr.append("\r\n");

        // store the request in the metadata?
        if (conf.getBoolean("store.http.request", false) == true) {
            headers.add("_request_", reqStr.toString());
        }

        byte[] reqBytes = reqStr.toString().getBytes();

        req.write(reqBytes);
        req.flush();

        LOG.info("Processing response..");

        PushbackInputStream in = // process response
                new PushbackInputStream(new BufferedInputStream(socket.getInputStream(), Http.BUFFER_SIZE),
                        Http.BUFFER_SIZE);

        StringBuffer line = new StringBuffer();

        // store the http headers verbatim
        if (conf.getBoolean("store.http.headers", false) == true) {
            httpHeaders = new StringBuffer();
        }

        headers.add("nutch.fetch.time", Long.toString(System.currentTimeMillis()));

        boolean haveSeenNonContinueStatus = false;
        while (!haveSeenNonContinueStatus) {
            // parse status code line
            this.code = parseStatusLine(in, line);
            if (httpHeaders != null)
                httpHeaders.append(line).append("\n");
            // parse headers
            parseHeaders(in, line, httpHeaders);
            haveSeenNonContinueStatus = code != 100; // 100 is "Continue"
        }

        if (httpHeaders != null) {
            headers.add("_response.headers_", httpHeaders.toString());
        }

        String transferEncoding = getHeader(Response.TRANSFER_ENCODING);
        LOG.info("Transfer Encoding for " + url + ":" + transferEncoding);
        if (transferEncoding != null && "chunked".equalsIgnoreCase(transferEncoding.trim())) {
            readChunkedContent(in, line);
        } else {
            readPlainContent(in);
        }

        String contentEncoding = getHeader(Response.CONTENT_ENCODING);
        if ("gzip".equals(contentEncoding) || "x-gzip".equals(contentEncoding)) {
            content = http.processGzipEncoded(content, url);
        } else if ("deflate".equals(contentEncoding)) {
            content = http.processDeflateEncoded(content, url);
        } else {
            if (Http.LOG.isTraceEnabled()) {
                Http.LOG.trace("fetched " + content.length + " bytes from " + url);
            }
        }

        LOG.info("Checking URL:" + url.toString());
        //check if url contains google drive string
        if (url.toString().toLowerCase().contains("https://drive.google.com/")) {
            //split into two string separated by '=' to get the article id
            LOG.info("Google Drive URL Detected!");
            String[] parts = url.toString().split("=");
            url = new URL("http://drive.google.com/uc?export=download&id=" + parts[1]);

            LOG.info("New URL:" + url.toString());
            this.http = http;
            this.url = url;
            this.orig = url.toString();
            this.base = url.toString();

            HttpClient client = new HttpClient();
            GetMethod method = new GetMethod(url.toString());
            int statusCode = client.executeMethod(method);
            content = method.getResponseBody();
            LOG.info("File Size on Drive: " + content.length);
            //   return;

        }

        LOG.info("Fetch Bytes: " + content.length + " bytes from " + url);

    } finally {
        if (socket != null)
            socket.close();
    }

}

From source file:org.forgerock.restlet.ext.oauth2.flow.AbstractFlow.java

public Set<String> getCheckedScope(String requestedScope, Set<String> maximumScope, Set<String> defaultScope) {
    if (null == requestedScope) {
        return defaultScope;
    } else {/*from www . j ava2 s.c  om*/
        Set<String> intersect = new TreeSet<String>(
                OAuth2Utils.split(requestedScope, OAuth2Utils.getScopeDelimiter(getContext())));
        Set<String> scopes = null;
        scopes = OAuth2Utils.parseScope(maximumScope);
        if (intersect.retainAll(scopes)) {
            OAuth2Utils.DEBUG.warning("AbstractFlow::Scope is different then requested");
            scopeChanged = true;
            return intersect;
        } else {
            scopeChanged = false;
            return intersect;
        }
    }
}

From source file:org.languagetool.gui.LanguageToolSupport.java

void reloadConfig() {
    //FIXME/*from w w w .j a va  2s.co m*/
    //if mother tongue changes then create new JLanguageTool instance

    boolean update = false;

    Language language = languageTool.getLanguage();
    languageTool = new MultiThreadedJLanguageTool(language, config.getMotherTongue(),
            new UserConfig(config.getConfigurableValues()));
    config.initStyleCategories(languageTool.getAllRules());

    Set<String> disabledRules = config.getDisabledRuleIds();
    if (disabledRules == null) {
        disabledRules = Collections.emptySet();
    }

    Set<String> common = new HashSet<>(disabledRules);
    common.retainAll(languageTool.getDisabledRules());
    Set<String> toDisable = new HashSet<>(disabledRules);
    toDisable.removeAll(common);
    Set<String> toEnable = new HashSet<>(languageTool.getDisabledRules());
    toEnable.removeAll(common);

    for (String ruleId : toDisable) {
        languageTool.disableRule(ruleId);
        update = true;
    }
    for (String ruleId : toEnable) {
        languageTool.enableRule(ruleId);
        update = true;
    }

    Set<String> disabledCategoryNames = config.getDisabledCategoryNames();
    if (disabledCategoryNames == null) {
        disabledCategoryNames = Collections.emptySet();
    }
    Set<CategoryId> disabledCategories = new HashSet<>();
    Map<CategoryId, Category> langCategories = languageTool.getCategories();

    for (CategoryId id : langCategories.keySet()) {
        String categoryName = langCategories.get(id).getName();
        if (disabledCategoryNames.contains(categoryName)) {
            disabledCategories.add(id);
        }
    }

    Set<CategoryId> ltDisabledCategories = new HashSet<>();
    for (CategoryId id : langCategories.keySet()) {
        if (languageTool.isCategoryDisabled(id)) {
            ltDisabledCategories.add(id);
        }
    }

    Set<CategoryId> commonCat = new HashSet<>(disabledCategories);
    commonCat.retainAll(ltDisabledCategories);

    Set<CategoryId> toDisableCat = new HashSet<>(disabledCategories);
    toDisableCat.removeAll(commonCat);

    Set<CategoryId> toEnableCat = new HashSet<>(ltDisabledCategories);
    toEnableCat.removeAll(commonCat);

    for (CategoryId id : toDisableCat) {
        languageTool.disableCategory(id);
    }
    for (CategoryId id : toEnableCat) {
        languageTool.enableRuleCategory(id);
    }
    if (!toDisableCat.isEmpty() || !toEnableCat.isEmpty()) {
        // ugly hack to trigger reInitSpellCheckIgnoreWords()
        update = true;
    }

    Set<String> enabledRules = config.getEnabledRuleIds();
    if (enabledRules == null) {
        enabledRules = Collections.emptySet();
    }
    for (String ruleName : enabledRules) {
        languageTool.enableRule(ruleName);
        update = true;
    }

    //    languageTool.setConfigValues(config.getConfigValues());

    if (update) {
        //FIXME
        //we could skip a full check if the user disabled but didn't enable rules
        checkImmediately(null);
        fireEvent(LanguageToolEvent.Type.RULE_ENABLED, null);
    }
}

From source file:org.geoserver.geopkg.GeoPackageGetMapOutputFormat.java

GridSubset findBestGridSubset(WMSMapContent map) {
    GetMapRequest req = map.getRequest();
    Map formatOpts = req.getFormatOptions();

    GridSetBroker gridSetBroker = gwc.getGridSetBroker();
    GridSet gridSet = null;// w  w  w.  j a va2 s  .c  o  m

    //first check format options to see if explicitly specified
    if (formatOpts.containsKey("gridset")) {
        gridSet = gridSetBroker.get(formatOpts.get("gridset").toString());
    }

    //next check srs
    if (gridSet == null) {
        gridSet = gridSetBroker.get(req.getSRS().toUpperCase());
    }

    if (gridSet != null) {
        return GridSubsetFactory.createGridSubSet(gridSet);
    }

    CoordinateReferenceSystem crs = map.getCoordinateReferenceSystem();

    //look up epsg code
    Integer epsgCode = null;
    try {
        epsgCode = CRS.lookupEpsgCode(crs, false);
    } catch (Exception e) {
        throw new ServiceException("Unable to determine epsg code for " + crs, e);
    }
    if (epsgCode == null) {
        throw new ServiceException("Unable to determine epsg code for " + crs);
    }

    SRS srs = SRS.getSRS(epsgCode);

    //figure out the appropriate grid sub set
    Set<GridSubset> gridSubsets = new LinkedHashSet<GridSubset>();
    for (MapLayerInfo l : req.getLayers()) {
        TileLayer tl = gwc.getTileLayerByName(l.getName());
        if (tl == null) {
            throw new ServiceException("No tile layer for " + l.getName());
        }

        List<GridSubset> theseGridSubsets = tl.getGridSubsetsForSRS(srs);
        if (gridSubsets.isEmpty()) {
            gridSubsets.addAll(theseGridSubsets);
        } else {
            gridSubsets.retainAll(theseGridSubsets);
        }

        if (gridSubsets.isEmpty()) {
            throw new ServiceException("No suitable " + epsgCode + " grid subset for " + req.getLayers());
        }
    }

    if (gridSubsets.size() > 1) {
        if (LOGGER.isLoggable(Level.WARNING)) {
            StringBuilder msg = new StringBuilder("Found multiple grid subsets: ");
            for (GridSubset gs : gridSubsets) {
                msg.append(gs.getName()).append(", ");
            }
            msg.setLength(msg.length() - 2);
            msg.append(". Choosing first.");
            LOGGER.warning(msg.toString());
        }
    }

    return gridSubsets.iterator().next();
}

From source file:hudson.logging.LogRecorder.java

@Restricted(NoExternalUse.class)
public AutoCompletionCandidates doAutoCompleteLoggerName(@QueryParameter String value) {
    if (value == null) {
        return new AutoCompletionCandidates();
    }//w  ww  .  ja  v a  2s.  com

    // get names of all actual loggers known to Jenkins
    Set<String> candidateNames = new LinkedHashSet<>(
            getAutoCompletionCandidates(Collections.list(LogManager.getLogManager().getLoggerNames())));

    for (String part : value.split("[ ]+")) {
        HashSet<String> partCandidates = new HashSet<>();
        String lowercaseValue = part.toLowerCase(Locale.ENGLISH);
        for (String loggerName : candidateNames) {
            if (loggerName.toLowerCase(Locale.ENGLISH).contains(lowercaseValue)) {
                partCandidates.add(loggerName);
            }
        }
        candidateNames.retainAll(partCandidates);
    }
    AutoCompletionCandidates candidates = new AutoCompletionCandidates();
    candidates.add(candidateNames.toArray(MemoryReductionUtil.EMPTY_STRING_ARRAY));
    return candidates;
}

From source file:opennlp.tools.fca.BasicLevelMetrics.java

public void categoryFeatureCollocation() {
    if (attributesExtent == null)
        this.setUp();

    ArrayList<Integer> attrExtent;
    Set<Integer> intersection;
    double sum = 0;
    int latticeSize = cl.conceptList.size();
    for (FormalConcept c : cl.conceptList) {
        sum = 0;/*www  . j  a  va2s . co  m*/
        for (int i = 0; i < cl.attributeCount; i++) {
            intersection = new HashSet<Integer>();
            intersection.addAll(c.extent);
            attrExtent = attributesExtent.get(i);
            intersection.retainAll(attrExtent);
            sum += (double) intersection.size() * 1. / attrExtent.size() * intersection.size()
                    / c.extent.size();
        }
        c.blCFC = Double.isNaN(sum) ? 0 : sum;
    }
}

From source file:org.apache.nutch.protocol.s2jh.HttpResponse.java

public HttpResponse(HttpBase http, URL url, WebPage page) throws ProtocolException, IOException {
    conf = http.getConf();/*from   w  w w .j a  va 2s. co m*/
    this.http = http;
    this.url = url;
    Scheme scheme = null;

    if ("http".equals(url.getProtocol())) {
        scheme = Scheme.HTTP;
    } else if ("https".equals(url.getProtocol())) {
        scheme = Scheme.HTTPS;
    } else {
        throw new HttpException("Unknown scheme (not http/https) for url:" + url);
    }

    if (Http.LOG.isTraceEnabled()) {
        Http.LOG.trace("fetching " + url);
    }

    String path = "".equals(url.getFile()) ? "/" : url.getFile();

    // some servers will redirect a request with a host line like
    // "Host: <hostname>:80" to "http://<hpstname>/<orig_path>"- they
    // don't want the :80...

    String host = url.getHost();
    int port;
    String portString;
    if (url.getPort() == -1) {
        if (scheme == Scheme.HTTP) {
            port = 80;
        } else {
            port = 443;
        }
        portString = "";
    } else {
        port = url.getPort();
        portString = ":" + port;
    }
    Socket socket = null;

    try {
        socket = new Socket(); // create the socket
        socket.setSoTimeout(http.getTimeout());

        // connect
        String sockHost = http.useProxy() ? http.getProxyHost() : host;
        int sockPort = http.useProxy() ? http.getProxyPort() : port;
        InetSocketAddress sockAddr = new InetSocketAddress(sockHost, sockPort);
        socket.connect(sockAddr, http.getTimeout());

        if (scheme == Scheme.HTTPS) {
            SSLSocketFactory factory = (SSLSocketFactory) SSLSocketFactory.getDefault();
            SSLSocket sslsocket = (SSLSocket) factory.createSocket(socket, sockHost, sockPort, true);
            sslsocket.setUseClientMode(true);

            // Get the protocols and ciphers supported by this JVM
            Set<String> protocols = new HashSet<String>(Arrays.asList(sslsocket.getSupportedProtocols()));
            Set<String> ciphers = new HashSet<String>(Arrays.asList(sslsocket.getSupportedCipherSuites()));

            // Intersect with preferred protocols and ciphers
            protocols.retainAll(http.getTlsPreferredProtocols());
            ciphers.retainAll(http.getTlsPreferredCipherSuites());

            sslsocket.setEnabledProtocols(protocols.toArray(new String[protocols.size()]));
            sslsocket.setEnabledCipherSuites(ciphers.toArray(new String[ciphers.size()]));

            sslsocket.startHandshake();
            socket = sslsocket;
        }

        if (sockAddr != null && conf.getBoolean("store.ip.address", false) == true) {
            String ipString = sockAddr.getAddress().getHostAddress(); // get the ip
                                                                      // address
            page.getMetadata().put(new Utf8("_ip_"), ByteBuffer.wrap(ipString.getBytes()));
        }

        Http.LOG.debug("HTTP fetching: " + url);
        // make request
        OutputStream req = socket.getOutputStream();

        StringBuffer reqStr = new StringBuffer("GET ");
        if (http.useProxy()) {
            reqStr.append(url.getProtocol() + "://" + host + portString + path);
        } else {
            reqStr.append(path);
        }

        reqStr.append(" HTTP/1.0\r\n");

        reqStr.append("Host: ");
        reqStr.append(host);
        reqStr.append(portString);
        reqStr.append("\r\n");

        reqStr.append("Accept-Encoding: x-gzip, gzip\r\n");

        reqStr.append("Accept: ");
        reqStr.append(this.http.getAccept());
        reqStr.append("\r\n");

        String userAgent = http.getUserAgent();
        if ((userAgent == null) || (userAgent.length() == 0)) {
            if (Http.LOG.isErrorEnabled()) {
                Http.LOG.error("User-agent is not set!");
            }
        } else {
            reqStr.append("User-Agent: ");
            reqStr.append(userAgent);
            reqStr.append("\r\n");
        }

        // if (page.isReadable(WebPage.Field.MODIFIED_TIME.getIndex())) {
        reqStr.append("If-Modified-Since: " + HttpDateFormat.toString(page.getModifiedTime()));
        reqStr.append("\r\n");
        // }
        reqStr.append("\r\n");

        byte[] reqBytes = reqStr.toString().getBytes();

        req.write(reqBytes);
        req.flush();

        PushbackInputStream in = // process response
                new PushbackInputStream(new BufferedInputStream(socket.getInputStream(), Http.BUFFER_SIZE),
                        Http.BUFFER_SIZE);

        StringBuffer line = new StringBuffer();

        boolean haveSeenNonContinueStatus = false;
        while (!haveSeenNonContinueStatus) {
            // parse status code line
            this.code = parseStatusLine(in, line);
            // parse headers
            parseHeaders(in, line);
            haveSeenNonContinueStatus = code != 100; // 100 is "Continue"
        }

        if (!url.toString().endsWith("robots.txt")) {
            if (readPlainContent(url.toString(), in)) {
            } else if (readPlainContentByHtmlunit(url)) {
            } else {
                readPlainContentByWebDriver(url);
            }
        }

        if (content != null && content.length > 0) {
            String html = charset == null ? new String(content) : new String(content, charset);
            //System.out.println("URL: " + url + ", CharsetName: " + charset + " , Page HTML=\n" + html);
            Http.LOG_HTML.trace("URL: " + url + ", CharsetName: " + charset + " , Page HTML=\n" + html);
        }

        // add headers in metadata to row
        if (page.getHeaders() != null) {
            page.getHeaders().clear();
        }
        for (String key : headers.names()) {
            page.getHeaders().put(new Utf8(key), new Utf8(headers.get(key)));
        }

    } catch (Exception e) {
        Http.LOG.error(e.getMessage(), e);
    } finally {
        if (socket != null)
            socket.close();
    }

}