List of usage examples for org.apache.commons.httpclient MultiThreadedHttpConnectionManager MultiThreadedHttpConnectionManager
public MultiThreadedHttpConnectionManager()
From source file:com.ning.http.client.providers.apache.ApacheAsyncHttpProvider.java
public ApacheAsyncHttpProvider(AsyncHttpClientConfig config) { this.config = config; connectionManager = new MultiThreadedHttpConnectionManager(); params = new HttpClientParams(); params.setParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, Boolean.TRUE); params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY); params.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler()); AsyncHttpProviderConfig<?, ?> providerConfig = config.getAsyncHttpProviderConfig(); if (providerConfig != null && ApacheAsyncHttpProvider.class.isAssignableFrom(providerConfig.getClass())) { configure(ApacheAsyncHttpProviderConfig.class.cast(providerConfig)); }/*from w w w.ja v a 2 s . c o m*/ }
From source file:com.fluidops.iwb.provider.CkanProvider.java
@Override public void gather(List<Statement> res) throws Exception { // Read CKAN location and establish connection URL registryUrl = new URL(config.location); HttpURLConnection registryConnection = (HttpURLConnection) registryUrl.openConnection(); registryConnection.setRequestMethod("GET"); // Check if connection to CKAN could be established if (registryConnection.getResponseCode() != HttpURLConnection.HTTP_OK) { String msg = String.format("Connection to the CKAN registry could not be established. (%s, %s)", registryConnection.getResponseCode(), registryConnection.getResponseMessage()); logger.info(msg);/*from w ww . j a v a 2s. c om*/ throw new IllegalStateException(msg); } logger.trace("Connection to CKAN established successfully."); String siteContent = GenUtil.readUrl(registryConnection.getInputStream()); JSONObject groupAsJson = null; JSONArray packageListJsonArray = null; try { groupAsJson = new JSONObject(new JSONTokener(siteContent)); packageListJsonArray = groupAsJson.getJSONArray("packages"); } catch (JSONException e) { String msg = String.format("Returned content %s is not valid JSON. Check if the registry URL is valid.", siteContent); logger.debug(msg); throw new IllegalStateException(msg); } logger.trace("Extracted JSON from CKAN successfully"); // Create metadata about LOD catalog res.add(ProviderUtils.createStatement(CKAN.CKAN_CATALOG, RDF.TYPE, Vocabulary.DCAT.CATALOG)); res.add(ProviderUtils.createStatement(CKAN.CKAN_CATALOG, RDFS.LABEL, CKAN.CKAN_CATALOG_LABEL)); // Extract metadata for individual data sets listed in CKAN MultiThreadedHttpConnectionManager connectionManager = null; ExecutorService pool = null; try { pool = Executors.newFixedThreadPool(10); connectionManager = new MultiThreadedHttpConnectionManager(); HttpClient client = new HttpClient(connectionManager); List<Statement> synchedList = Collections.synchronizedList(res); for (int i = 0; i < packageListJsonArray.length(); i++) { String host = "http://www.ckan.net/package/" + packageListJsonArray.get(i).toString(); String baseUri = findBaseUri( "http://www.ckan.net/api/rest/package/" + packageListJsonArray.get(i).toString()); baseUri = (baseUri == null) ? host : baseUri; pool.execute(new MetadataReader(client, host, baseUri, CKAN.CKAN_CATALOG, synchedList)); } } finally { if (pool != null) { pool.shutdown(); pool.awaitTermination(4, TimeUnit.HOURS); } if (connectionManager != null) connectionManager.shutdown(); } }
From source file:net.sf.j2ep.ProxyFilter.java
/** * @see javax.servlet.Filter#init(javax.servlet.FilterConfig) * /*www .j a v a2 s.co m*/ * Called upon initialization, Will create the ConfigParser and get the * RuleChain back. Will also configure the httpclient. */ public void init(FilterConfig filterConfig) throws ServletException { log = LogFactory.getLog(ProxyFilter.class); AllowedMethodHandler.setAllowedMethods("OPTIONS,GET,HEAD,POST,PUT,DELETE,TRACE"); httpClient = new HttpClient(new MultiThreadedHttpConnectionManager()); httpClient.getParams().setBooleanParameter(HttpClientParams.USE_EXPECT_CONTINUE, false); httpClient.getParams().setCookiePolicy(CookiePolicy.IGNORE_COOKIES); String data = filterConfig.getInitParameter("dataUrl"); if (data == null) { serverChain = null; } else { try { File dataFile = new File(filterConfig.getServletContext().getRealPath(data)); ConfigParser parser = new ConfigParser(dataFile); serverChain = parser.getServerChain(); } catch (Exception e) { throw new ServletException(e); } } }
From source file:edu.harvard.med.iccbl.screensaver.soaputils.PugSoapUtil.java
private static void updateStub(org.apache.axis2.client.Stub stub, int maxTotal, int maxPerHost) { MultiThreadedHttpConnectionManager httpConnectionManager = new MultiThreadedHttpConnectionManager(); HttpConnectionManagerParams params = httpConnectionManager.getParams(); if (params == null) { params = new HttpConnectionManagerParams(); }//from w w w .j a v a2 s . com params.setMaxTotalConnections(maxTotal); params.setDefaultMaxConnectionsPerHost(maxPerHost); // extra, not prescribed -sde4 params.setConnectionTimeout(600000); params.setSoTimeout(600000); httpConnectionManager.setParams(params); HttpClient httpClient = new HttpClient(httpConnectionManager); ServiceClient serviceClient = stub._getServiceClient(); ConfigurationContext context = serviceClient.getServiceContext().getConfigurationContext(); context.setProperty(HTTPConstants.CACHED_HTTP_CLIENT, httpClient); }
From source file:com.netflix.genie.security.saml.SAMLConfig.java
/** * Connection pool for the HTTP Client./*from w ww. ja v a 2 s . co m*/ * * @return a Multithreaded connection manager * @see MultiThreadedHttpConnectionManager */ @Bean public MultiThreadedHttpConnectionManager multiThreadedHttpConnectionManager() { return new MultiThreadedHttpConnectionManager(); }
From source file:com.noelios.restlet.ext.httpclient.HttpClientHelper.java
@Override public void start() throws Exception { super.start(); // Create the multi-threaded connection manager and configure it final MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager(); connectionManager.getParams().setConnectionTimeout(getConnectTimeout()); connectionManager.getParams().setDefaultMaxConnectionsPerHost(getMaxConnectionsPerHost()); connectionManager.getParams().setMaxTotalConnections(getMaxTotalConnections()); connectionManager.getParams().setTcpNoDelay(getTcpNoDelay()); // Create the internal client connector this.httpClient = new HttpClient(connectionManager); getHttpClient().getParams().setAuthenticationPreemptive(false); getHttpClient().getParams().setConnectionManagerTimeout(getConnectionManagerTimeout()); getHttpClient().getParams().setCookiePolicy(CookiePolicy.IGNORE_COOKIES); getHttpClient().getParams().setSoTimeout(getReadTimeout()); getLogger().info("Starting the HTTP client"); }
From source file:com.naveen.demo.config.Saml2SSOConfig.java
@Bean public MultiThreadedHttpConnectionManager multiThreadedHttpConnectionManager() { return new MultiThreadedHttpConnectionManager(); }
From source file:com.fluidops.iwb.HTMLProvider.HTMLProvider.java
/** * HINT: The gather(List<Statement> res) method collects the statements * extracted by the provider. Use the following guidelinges: * /*from w w w .j av a 2 s .com*/ * 1.) Make sure to have a clear documentation, structure, and * modularization. Use helper methods wherever possible to increase * readability of the method. * * 2.) Whenever there is a need to create statements, use the helper methods * in {@link ProviderUtils}. This class helps you in generating "safe" URIs, * replacing invalid characters etc. It also offers common functionality for * filtering statements, e.g. removing statements containing null values. * * 3.) Re-use existing ontologies! The {@link Vocabulary} class provides a * mix of vocabulary from common ontologies and can be easily extended. You * should not define URIs inside the provider itself, except these URIs are * absolutely provider-specific. * * 4.) Concerning exception handling, it is best practice to throw * exceptions whenever the provider run cannot be finished in a regular way. * Since these exception will be propagated to the UI, it is recommended to * catch Exceptions locally first, log them, and wrap them into * (Runtime)Exceptions with a human-readable description. When logging * exceptions, the log level "warn" is appropriate. */ @Override public void gather(List<Statement> res) throws Exception { URL registryUrl = new URL(config.location); HttpURLConnection registryConnection = (HttpURLConnection) registryUrl.openConnection(); registryConnection.setRequestMethod("GET"); // ////////////////////////////////////////////////////////////////////// // /////////////////////////////////////////////////////////////// STEP // 1 logger.info("Retrieving packages from CKAN..."); if (registryConnection.getResponseCode() != HttpURLConnection.HTTP_OK) { String msg = "Connection with the registry could not be established. (" + registryConnection.getResponseCode() + ", " + registryConnection.getResponseMessage() + ")"; logger.warn(msg); throw new RuntimeException(msg); // propagate to UI } String siteContent = GenUtil.readUrl(registryConnection.getInputStream()); JSONObject groupAsJson = null; JSONArray packageListJsonArray = null; try { groupAsJson = new JSONObject(new JSONTokener(siteContent)); packageListJsonArray = groupAsJson.getJSONArray("packages"); } catch (JSONException e) { String msg = "Returned content " + siteContent + " is not valid JSON. Check if the registry URL is valid."; logger.warn(msg); throw new RuntimeException(msg); // propagate to UI } logger.info("-> found " + packageListJsonArray.length() + " packages"); // ////////////////////////////////////////////////////////////////////// // /////////////////////////////////////////////////////////////// STEP // 2 logger.info("Registering LOD catalog in metadata repository"); /** * HINT: the method createStatement allows to create statements if * subject, predicate and object are all known; use this method instead * of opening a value factory */ res.add(ProviderUtils.createStatement(CKANVocabulary.CKAN_CATALOG, RDF.TYPE, Vocabulary.DCAT.CATALOG)); res.add(ProviderUtils.createStatement(CKANVocabulary.CKAN_CATALOG, RDFS.LABEL, CKANVocabulary.CKAN_CATALOG_LABEL)); logger.info("-> done"); // ////////////////////////////////////////////////////////////////////// // /////////////////////////////////////////////////////////////// STEP // 3 logger.info("Extracting metdata for the individual data sets listed in CKAN"); /** * HINT: Set up an Apache HTTP client with a manager for multiple * threads; as a general guideline, use parallelization whenever * crawling web sources! */ MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager(); HttpClient client = new HttpClient(connectionManager); ExecutorService pool = Executors.newFixedThreadPool(10); // we store the data in a temporary memory store, which allows us // to perform transformation on the result set Repository repository = null; RepositoryConnection connection = null; try { // initialize repository and connection repository = new SailRepository(new MemoryStore()); repository.initialize(); connection = repository.getConnection(); // Fire up a thread for every package logger.info("-> Fire up threads for the individual packages..."); for (int i = 0; i < packageListJsonArray.length(); i++) { // we use the JSON representation to get a base URI to resolve // relative // URIs in the XML later on. (and a fallback solution) String host = "http://www.ckan.net/package/" + packageListJsonArray.get(i).toString(); String baseUri = findBaseUri( "http://www.ckan.net/api/rest/package/" + packageListJsonArray.get(i).toString()); baseUri = (baseUri == null) ? host : baseUri; pool.execute(new MetadataReader(client, host, baseUri, CKANVocabulary.CKAN_CATALOG, connection)); } logger.info("-> Waiting for all tasks to complete (" + packageListJsonArray.length() + "tasks/data sources)..."); pool.shutdown(); pool.awaitTermination(4, TimeUnit.HOURS); /** * Now the extraction has finished, all statements are available in * our temporary repository. We apply some conversions and * transformations to align the extracted statements with our target * ontology. * * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! * !!!!!!!!!!!!! !!! NOTE: this code is /NOT/ best practice, we * should eventually extend !!! !!! ProviderUtils to deal with at * least lightweight transformations !!! !!! (such as changing * property names) or realize such tasks using !!! !!! an integrated * mapping framework. !!! * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! */ // Extraction from temporary repository, phase 1: logger.info( "-> Extract dcterms:title AS rdfs:label, dcterms:contributor AS dcterms:creator, and dcterms:rights AS dcterms:license"); String mappingQuery = mappingQuery(); GraphQuery mappingGraphQuery = connection.prepareGraphQuery(QueryLanguage.SPARQL, mappingQuery); GraphQueryResult result = mappingGraphQuery.evaluate(); logger.info("-> Appending extracted result to statement list"); ProviderUtils.appendGraphQueryResultToListAndClose(result, res); // Label the distribution nodes logger.info("-> Generate labels for distributions"); String labelDistributionQuery = labelDistributionQuery(); GraphQuery labelDistributionGraphQuery = connection.prepareGraphQuery(QueryLanguage.SPARQL, labelDistributionQuery); GraphQueryResult result2 = labelDistributionGraphQuery.evaluate(); logger.info("-> Appending extracted result to statement list"); ProviderUtils.appendGraphQueryResultToListAndClose(result2, res); // Extraction from temporary repository, phase 2: logger.info("-> Deleting previously extracted triples and additional, not required information..."); String deleteQuery = deleteQuery(); Update deleteGraphQuery = connection.prepareUpdate(QueryLanguage.SPARQL, deleteQuery); deleteGraphQuery.execute(); // Extraction from temporary repository, phase 3: logger.info("-> Deleting dcat:distribution and dcat:accessUrl information from" + "temp repository for which format information is missing..."); String cleanDistQuery = cleanDistQuery(); Update cleanupGraphQuery = connection.prepareUpdate(QueryLanguage.SPARQL, cleanDistQuery); cleanupGraphQuery.execute(); logger.info("-> Appending remaining statements to result..."); connection.getStatements(null, null, null, false).addTo(res); logger.info("Provider run finished successfully"); } catch (Exception e) { logger.warn(e.getMessage()); throw new RuntimeException(e); } finally { if (connection != null) connection.close(); if (repository != null) repository.shutDown(); } // in the end, make sure there are no statements containing null in // any of the position (did not take special care when creating // statements) logger.info("-> cleaning up null statements"); res = ProviderUtils.filterNullStatements(res); }
From source file:com.ning.http.client.providers.apache.TestableApacheAsyncHttpProvider.java
public TestableApacheAsyncHttpProvider(AsyncHttpClientConfig config) { super(config); this.config = config; connectionManager = new MultiThreadedHttpConnectionManager(); params = new HttpClientParams(); params.setParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, Boolean.TRUE); params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY); params.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler()); }
From source file:fr.eurecom.nerd.core.proxy.ExtractivClient.java
private static HttpClient getClient() { if (httpClient == null) { // snuff the getBody() HTTPClient warnings final Logger logger = Logger.getLogger("org.apache.commons.httpclient"); logger.setLevel(Level.SEVERE); httpClient = new HttpClient(new MultiThreadedHttpConnectionManager()); }/*from w w w. jav a 2 s . c o m*/ return httpClient; }