List of usage examples for org.apache.commons.httpclient MultiThreadedHttpConnectionManager MultiThreadedHttpConnectionManager
public MultiThreadedHttpConnectionManager()
From source file:org.opensaml.ws.soap.client.http.HttpClientBuilder.java
/** * Builds an HTTP client with the given settings. Settings are NOT reset to their default values after a client has * been created.//from w w w . j av a 2s . c om * * @return the created client. */ public HttpClient buildClient() { if (httpsProtocolSocketFactory != null) { Protocol.registerProtocol("https", new Protocol("https", httpsProtocolSocketFactory, 443)); } HttpClientParams clientParams = new HttpClientParams(); clientParams.setAuthenticationPreemptive(isPreemptiveAuthentication()); clientParams.setContentCharset(getContentCharSet()); clientParams.setParameter(HttpClientParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(connectionRetryAttempts, false)); HttpConnectionManagerParams connMgrParams = new HttpConnectionManagerParams(); connMgrParams.setConnectionTimeout(getConnectionTimeout()); connMgrParams.setDefaultMaxConnectionsPerHost(getMaxConnectionsPerHost()); connMgrParams.setMaxTotalConnections(getMaxTotalConnections()); connMgrParams.setReceiveBufferSize(getReceiveBufferSize()); connMgrParams.setSendBufferSize(getSendBufferSize()); connMgrParams.setTcpNoDelay(isTcpNoDelay()); MultiThreadedHttpConnectionManager connMgr = new MultiThreadedHttpConnectionManager(); connMgr.setParams(connMgrParams); HttpClient httpClient = new HttpClient(clientParams, connMgr); if (proxyHost != null) { HostConfiguration hostConfig = new HostConfiguration(); hostConfig.setProxy(proxyHost, proxyPort); httpClient.setHostConfiguration(hostConfig); if (proxyUsername != null) { AuthScope proxyAuthScope = new AuthScope(proxyHost, proxyPort); UsernamePasswordCredentials proxyCredentials = new UsernamePasswordCredentials(proxyUsername, proxyPassword); httpClient.getState().setProxyCredentials(proxyAuthScope, proxyCredentials); } } return httpClient; }
From source file:org.opensaml.ws.soap.client.HTTPSOAPTransportFactory.java
/** * Initializes the {@link HttpClient} that will be used by the created {@link HTTPSOAPTransport} built by this * factory.//from w w w . jav a2 s . c om */ protected void initializeHttpClient() { HttpConnectionManagerParams connectionParams = new HttpConnectionManagerParams(); connectionParams.setConnectionTimeout(connectionTimeout); MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager(); connectionManager.setParams(connectionParams); HttpClientParams clientParams = new HttpClientParams(); clientParams.setSoTimeout(socketTimeout); if (httpVersion == HTTP_VERSION.HTTP1_0) { clientParams.setVersion(HttpVersion.HTTP_1_0); } else { clientParams.setVersion(HttpVersion.HTTP_1_1); } httpClient = new HttpClient(clientParams, connectionManager); }
From source file:org.parosproxy.paros.network.HttpSender.java
private HttpClient createHttpClient() { httpConnManager = new MultiThreadedHttpConnectionManager(); setCommonManagerParams(httpConnManager); return new HttpClient(httpConnManager); }
From source file:org.parosproxy.paros.network.HttpSender.java
private HttpClient createHttpClientViaProxy() { if (!param.isUseProxyChain()) { return createHttpClient(); }/* ww w. j ava 2 s . c om*/ httpConnManagerProxy = new MultiThreadedHttpConnectionManager(); setCommonManagerParams(httpConnManagerProxy); HttpClient clientProxy = new HttpClient(httpConnManagerProxy); clientProxy.getHostConfiguration().setProxy(param.getProxyChainName(), param.getProxyChainPort()); if (param.isUseProxyChainAuth()) { clientProxy.getState().setProxyCredentials(getAuthScope(param), getNTCredentials(param)); } return clientProxy; }
From source file:org.paxle.crawler.http.impl.HttpCrawler.java
@Modified public synchronized void modified(Map<String, Object> configuration) { /*/*w w w . j av a 2 s .c o m*/ * Cleanup old config */ this.cleanup(); /* * Init with changed configuration */ this.connectionManager = new MultiThreadedHttpConnectionManager(); final HttpConnectionManagerParams connectionManagerParams = connectionManager.getParams(); // configure connections per host final Integer maxConnections = (Integer) configuration.get(PROP_MAXCONNECTIONS_PER_HOST); if (maxConnections != null) { connectionManagerParams.setDefaultMaxConnectionsPerHost(maxConnections.intValue()); } // configuring timeouts final Integer connectionTimeout = (Integer) configuration.get(PROP_CONNECTION_TIMEOUT); if (connectionTimeout != null) { connectionManagerParams.setConnectionTimeout(connectionTimeout.intValue()); } final Integer socketTimeout = (Integer) configuration.get(PROP_SOCKET_TIMEOUT); if (socketTimeout != null) { connectionManagerParams.setSoTimeout(socketTimeout.intValue()); } // set new http client this.httpClient = new HttpClient(connectionManager); // the crawler should request and accept content-encoded data final Boolean acceptEncoding = (Boolean) configuration.get(PROP_ACCEPT_ENCODING); if (acceptEncoding != null) { this.acceptEncoding = acceptEncoding.booleanValue(); } // specifies if the crawler should skipp unsupported-mime-types final Boolean skipUnsupportedMimeTypes = (Boolean) configuration.get(PROP_SKIP_UNSUPPORTED_MIMETYPES); if (skipUnsupportedMimeTypes != null) { this.skipUnsupportedMimeTypes = skipUnsupportedMimeTypes.booleanValue(); } // the cookie policy to use for crawling final String propCookiePolicy = (String) configuration.get(PROP_COOKIE_POLICY); this.cookiePolicy = (propCookiePolicy == null || propCookiePolicy.length() == 0) ? CookiePolicy.BROWSER_COMPATIBILITY : propCookiePolicy; // the http-user-agent string that should be used final String userAgent = (String) configuration.get(PROP_USER_AGENT); if (userAgent != null) { StringBuffer buf = new StringBuffer(); Pattern pattern = Pattern.compile("\\$\\{[^\\}]*}"); Matcher matcher = pattern.matcher(userAgent); // replacing property placeholders with system-properties while (matcher.find()) { String placeHolder = matcher.group(); String propName = placeHolder.substring(2, placeHolder.length() - 1); String propValue = System.getProperty(propName); if (propValue != null) matcher.appendReplacement(buf, propValue); } matcher.appendTail(buf); this.userAgent = buf.toString(); } else { // Fallback this.userAgent = "PaxleFramework"; } // download limit in bytes final Integer maxDownloadSize = (Integer) configuration.get(PROP_MAXDOWNLOAD_SIZE); if (maxDownloadSize != null) { this.maxDownloadSize = maxDownloadSize.intValue(); } // limit data transfer rate final Integer transferLimit = (Integer) configuration.get(PROP_TRANSFER_LIMIT); int limitKBps = 0; if (transferLimit != null) limitKBps = transferLimit.intValue(); this.logger.debug("transfer rate limit: " + limitKBps + " kb/s"); // TODO: lrc = (limitKBps > 0) ? new CrawlerTools.LimitedRateCopier(limitKBps) : null; // proxy configuration final Boolean useProxyVal = (Boolean) configuration.get(PROP_PROXY_USE); final String host = (String) configuration.get(PROP_PROXY_HOST); final Integer portVal = (Integer) configuration.get(PROP_PROXY_PORT); if (useProxyVal != null && useProxyVal.booleanValue() && host != null && host.length() > 0 && portVal != null) { this.logger.info(String.format("Proxy is enabled: %s:%d", host, portVal)); final int port = portVal.intValue(); final ProxyHost proxyHost = new ProxyHost(host, port); this.httpClient.getHostConfiguration().setProxyHost(proxyHost); final String user = (String) configuration.get(PROP_PROXY_HOST); final String pwd = (String) configuration.get(PROP_PROXY_PASSWORD); if (user != null && user.length() > 0 && pwd != null && pwd.length() > 0) this.httpClient.getState().setProxyCredentials(new AuthScope(host, port), new UsernamePasswordCredentials(user, pwd)); } else { this.logger.info("Proxy is disabled"); this.httpClient.getHostConfiguration().setProxyHost(null); this.httpClient.getState().clearCredentials(); } }
From source file:org.paxle.crawler.urlRedirector.impl.testers.HttpTester.java
protected void activate(Map<String, Object> props) throws UnknownHostException, IOException { // init http-client this.connectionManager = new MultiThreadedHttpConnectionManager(); this.httpClient = new HttpClient(this.connectionManager); }
From source file:org.paxle.filter.robots.impl.RobotsTxtManager.java
void init(Map<String, Object> props) { // configure caching manager this.manager = CacheManager.getInstance(); /* ================================================================================= * init a new cache//from ww w. j av a 2s. co m * ================================================================================= */ Integer maxCacheSize = (Integer) props.get(PROP_MAX_CACHE_SIZE); if (maxCacheSize == null) maxCacheSize = Integer.valueOf(1000); this.cache = new Cache(CACHE_NAME, maxCacheSize.intValue(), false, false, 60 * 60, 30 * 60); this.manager.addCache(this.cache); /* ================================================================================= * init threadpool * ================================================================================= */ Integer maxIdle = (Integer) props.get(PROP_WORKER_MAX_IDLE); if (maxIdle == null) maxIdle = Integer.valueOf(20); Integer maxAlive = (Integer) props.get(PROP_WORKER_MAX_ALIVE); if (maxAlive == null) maxAlive = Integer.valueOf(20); if (maxAlive.compareTo(maxIdle) < 0) maxAlive = maxIdle; this.execService = new ThreadPoolExecutor(maxIdle.intValue(), maxAlive.intValue(), 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>()); /* ================================================================================= * init http-client * ================================================================================= */ this.connectionManager = new MultiThreadedHttpConnectionManager(); HttpConnectionManagerParams params = this.connectionManager.getParams(); final Integer connectionTimeout = (Integer) props.get(PROP_CONNECTION_TIMEOUT); if (connectionTimeout != null) params.setConnectionTimeout(connectionTimeout.intValue()); final Integer socketTimeout = (Integer) props.get(PROP_SOCKET_TIMEOUT); if (socketTimeout != null) params.setSoTimeout(socketTimeout.intValue()); final Integer maxConnections = (Integer) props.get(PROP_MAXCONNECTIONS_TOTAL); if (maxConnections != null) params.setMaxTotalConnections(maxConnections.intValue()); this.httpClient = new HttpClient(this.connectionManager); /* ================================================================================= * proxy configuration * ================================================================================= */ final Boolean useProxyVal = (Boolean) props.get(PROP_PROXY_USE); final boolean useProxy = (useProxyVal == null) ? false : useProxyVal.booleanValue(); final String host = (String) props.get(PROP_PROXY_HOST); final Integer portVal = (Integer) props.get(PROP_PROXY_PORT); if (useProxy && host != null && host.length() > 0 && portVal != null) { final int port = portVal.intValue(); this.logger.info(String.format("Proxy is enabled: %s:%d", host, Integer.valueOf(port))); final ProxyHost proxyHost = new ProxyHost(host, port); this.httpClient.getHostConfiguration().setProxyHost(proxyHost); final String user = (String) props.get(PROP_PROXY_HOST); final String pwd = (String) props.get(PROP_PROXY_PASSWORD); if (user != null && user.length() > 0 && pwd != null && pwd.length() > 0) this.httpClient.getState().setProxyCredentials(new AuthScope(host, port), new UsernamePasswordCredentials(user, pwd)); } else { this.logger.info("Proxy is disabled"); this.httpClient.getHostConfiguration().setProxyHost(null); this.httpClient.getState().clearCredentials(); } /* ================================================================================= * the user-agent name that should be used * ================================================================================= */ final String userAgent = (String) props.get(PROP_USER_AGENT); if (userAgent != null) { final StringBuffer buf = new StringBuffer(); Pattern pattern = Pattern.compile("\\$\\{[^\\}]*}"); Matcher matcher = pattern.matcher(userAgent); // replacing property placeholders with system-properties while (matcher.find()) { String placeHolder = matcher.group(); String propName = placeHolder.substring(2, placeHolder.length() - 1); String propValue = System.getProperty(propName); if (propValue != null) matcher.appendReplacement(buf, propValue); } matcher.appendTail(buf); this.userAgent = buf.toString(); } else { // Fallback this.userAgent = "PaxleFramework"; } this.logger .info(String.format("Robots.txt manager initialized. Using '%s' rule-store with %d stored entries.", loader.getClass().getSimpleName(), Integer.valueOf(loader.size()))); }
From source file:org.pentaho.di.cluster.SlaveConnectionManager.java
private SlaveConnectionManager() { manager = new MultiThreadedHttpConnectionManager(); manager.getParams().setDefaultMaxConnectionsPerHost(100); manager.getParams().setMaxTotalConnections(200); }
From source file:org.portletbridge.portlet.DefaultHttpClientTemplate.java
/** * //from ww w . j a v a 2 s . c om */ private DefaultHttpClientTemplate(Builder b) { retryCount = b.retryCount; connectionTimeout = b.connectionTimeout; readTimeoutMillis = b.readTimeoutMillis; httpClient = new HttpClient(new MultiThreadedHttpConnectionManager()); httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(connectionTimeout); httpClient.getHttpConnectionManager().getParams().setSoTimeout(readTimeoutMillis); HttpMethodRetryHandler retryhandler = new HttpMethodRetryHandler() { public boolean retryMethod(final HttpMethod method, final IOException exception, int executionCount) { if (executionCount >= retryCount) { // Do not retry if over max retry count return false; } if (exception instanceof NoHttpResponseException) { // Retry if the server dropped connection on us return true; } if (exception instanceof SocketException) { // Retry if the server reset connection on us return true; } if (exception instanceof SocketTimeoutException) { // Retry if the read timed out return true; } if (!method.isRequestSent()) { // Retry if the request has not been sent fully or // if it's OK to retry methods that have been sent return true; } // otherwise do not retry return false; } }; httpClient.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryhandler); }
From source file:org.red5.server.net.servlet.AMFTunnelServlet.java
/** * Redirect to HTTP port.//from w ww. j a v a 2s . c o m */ @Override protected void service(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { HttpClient client = new HttpClient(new MultiThreadedHttpConnectionManager()); client.getHttpConnectionManager().getParams().setConnectionTimeout(30000); PostMethod get = new PostMethod("http://localhost:8080/gateway"); try { // copy all the headers // Enumeration headerNames = req.getHeaderNames(); // while (headerNames.hasMoreElements()) { // String headerName = (String) headerNames.nextElement(); // logger.debug("Adding received header to tunnel request: " // + headerName); // get.addRequestHeader(headerName, req.getHeader(headerName)); // } // Enumeration parameterNames = req.getParameterNames(); // while (parameterNames.hasMoreElements()) { // String parameterName = (String) parameterNames.nextElement(); // logger.debug("Adding received parameter to tunnel request: " // + parameterName); // get.getParams().setParameter(parameterName, // req.getParameter(parameterName)); // } // Enumeration attributeNames = req.getAttributeNames(); // while (attributeNames.hasMoreElements()) { // String attributeName = (String) attributeNames.nextElement(); // logger.debug("Adding received attribute to tunnel request: " // + attributeName); // } String path = req.getContextPath(); if (path == null) { path = ""; } // System.out.println("Path: " + path); if (req.getPathInfo() != null) { path += req.getPathInfo(); } // System.out.println("Path: " + path); int reqContentLength = req.getContentLength(); if (reqContentLength > 0) { // System.out.println("Request content length: " + // reqContentLength); ByteBuffer reqBuffer = ByteBuffer.allocate(reqContentLength); ServletUtils.copy(req.getInputStream(), reqBuffer.asOutputStream()); reqBuffer.flip(); get.setRequestEntity(new InputStreamRequestEntity(reqBuffer.asInputStream(), reqContentLength, "application/x-amf")); // get.setPath(path); get.addRequestHeader("Tunnel-request", path); client.executeMethod(get); // System.out.println("Response code: " + get.getStatusCode()); if (get.getStatusCode() == HttpStatus.SC_OK) { resp.setContentType("application/x-amf"); int responseLength = ((Long) get.getResponseContentLength()).intValue(); ByteBuffer respBuffer = ByteBuffer.allocate(responseLength); ServletUtils.copy(get.getResponseBodyAsStream(), respBuffer.asOutputStream()); respBuffer.flip(); ServletUtils.copy(respBuffer.asInputStream(), resp.getOutputStream()); resp.flushBuffer(); } else { resp.sendError(get.getStatusCode()); } } else { resp.sendError(HttpStatus.SC_BAD_REQUEST); } } catch (Exception ex) { ex.printStackTrace(); } finally { get.releaseConnection(); } }