Java tutorial
/* * This file is part of the Wayback archival access software * (http://archive-access.sourceforge.net/projects/wayback/). * * Licensed to the Internet Archive (IA) by one or more individual * contributors. * * The IA licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.archive.wayback.liveweb; import java.io.ByteArrayInputStream; import java.io.IOException; import java.net.ConnectException; import java.net.SocketException; import java.net.SocketTimeoutException; import java.net.URL; import java.util.logging.Logger; import java.util.zip.GZIPInputStream; import org.apache.commons.httpclient.ConnectTimeoutException; import org.apache.commons.httpclient.HostConfiguration; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.commons.httpclient.NoHttpResponseException; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.params.HttpClientParams; import org.archive.io.arc.ARCRecord; import org.archive.wayback.core.Resource; import org.archive.wayback.exception.LiveDocumentNotAvailableException; import org.archive.wayback.exception.LiveWebCacheUnavailableException; import org.archive.wayback.exception.LiveWebTimeoutException; import org.archive.wayback.exception.ResourceNotAvailableException; import org.archive.wayback.resourcestore.resourcefile.ArcResource; import org.archive.wayback.resourcestore.resourcefile.ResourceFactory; /** * This class fetches resource from live web. * It works with {@link ARCRecordingProxy} not standard proxy servers. * * @author brad * @see LiveWebCache * @see StdRemoteLiveWebCache * */ public class ArcRemoteLiveWebCache implements LiveWebCache { private static final Logger LOGGER = Logger.getLogger(ArcRemoteLiveWebCache.class.getName()); protected MultiThreadedHttpConnectionManager connectionManager = null; protected HostConfiguration hostConfiguration = null; protected HttpClient http = null; protected String requestPrefix = null; /** * */ public ArcRemoteLiveWebCache() { connectionManager = new MultiThreadedHttpConnectionManager(); hostConfiguration = new HostConfiguration(); HttpClientParams params = new HttpClientParams(); params.setParameter(HttpClientParams.RETRY_HANDLER, new NoRetryHandler()); http = new HttpClient(params, connectionManager); http.setHostConfiguration(hostConfiguration); } /* (non-Javadoc) * @see org.archive.wayback.liveweb.LiveWebCache#getCachedResource(java.net.URL, long, boolean) */ public Resource getCachedResource(URL url, long maxCacheMS, boolean bUseOlder) throws LiveDocumentNotAvailableException, LiveWebCacheUnavailableException, LiveWebTimeoutException, IOException { String urlString = url.toExternalForm(); if (requestPrefix != null) { urlString = requestPrefix + urlString; } HttpMethod method = null; try { method = new GetMethod(urlString); } catch (IllegalArgumentException e) { LOGGER.warning("Bad URL for live web fetch:" + urlString); throw new LiveDocumentNotAvailableException("Url:" + urlString + "does not look like an URL?"); } boolean success = false; try { int status = http.executeMethod(method); if (status == 200) { ByteArrayInputStream bais = new ByteArrayInputStream(method.getResponseBody()); ARCRecord r = new ARCRecord(new GZIPInputStream(bais), "id", 0L, false, false, true); ArcResource ar = (ArcResource) ResourceFactory.ARCArchiveRecordToResource(r, null); if (ar.getStatusCode() == 502) { throw new LiveDocumentNotAvailableException(urlString); } else if (ar.getStatusCode() == 504) { throw new LiveWebTimeoutException("Timeout:" + urlString); } success = true; return ar; } else { throw new LiveWebCacheUnavailableException(urlString); } } catch (ResourceNotAvailableException e) { throw new LiveDocumentNotAvailableException(urlString); } catch (NoHttpResponseException e) { throw new LiveWebCacheUnavailableException("No Http Response for " + urlString); } catch (ConnectException e) { throw new LiveWebCacheUnavailableException(e.getLocalizedMessage() + " : " + urlString); } catch (SocketException e) { throw new LiveWebCacheUnavailableException(e.getLocalizedMessage() + " : " + urlString); } catch (SocketTimeoutException e) { throw new LiveWebTimeoutException(e.getLocalizedMessage() + " : " + urlString); } catch (ConnectTimeoutException e) { throw new LiveWebTimeoutException(e.getLocalizedMessage() + " : " + urlString); } finally { if (!success) { method.abort(); } method.releaseConnection(); } } /* (non-Javadoc) * @see org.archive.wayback.liveweb.LiveWebCache#shutdown() */ public void shutdown() { // TODO Auto-generated method stub } /** * @param hostPort to proxy requests through - ex. "localhost:3128" */ public void setProxyHostPort(String hostPort) { int colonIdx = hostPort.indexOf(':'); if (colonIdx > 0) { String host = hostPort.substring(0, colonIdx); int port = Integer.valueOf(hostPort.substring(colonIdx + 1)); // http.getHostConfiguration().setProxy(host, port); hostConfiguration.setProxy(host, port); } } /** * @param maxTotalConnections the HttpConnectionManagerParams config */ public void setMaxTotalConnections(int maxTotalConnections) { connectionManager.getParams().setMaxTotalConnections(maxTotalConnections); } /** * @return the HttpConnectionManagerParams maxTotalConnections config */ public int getMaxTotalConnections() { return connectionManager.getParams().getMaxTotalConnections(); } /** * @param maxHostConnections the HttpConnectionManagerParams config */ public void setMaxHostConnections(int maxHostConnections) { connectionManager.getParams().setMaxConnectionsPerHost(hostConfiguration, maxHostConnections); } /** * @return the HttpConnectionManagerParams maxHostConnections config */ public int getMaxHostConnections() { return connectionManager.getParams().getMaxConnectionsPerHost(hostConfiguration); } /** * @return the connectionTimeoutMS */ public int getConnectionTimeoutMS() { return connectionManager.getParams().getConnectionTimeout(); } /** * @param connectionTimeoutMS the connectionTimeoutMS to set */ public void setConnectionTimeoutMS(int connectionTimeoutMS) { connectionManager.getParams().setConnectionTimeout(connectionTimeoutMS); } /** * @return the socketTimeoutMS */ public int getSocketTimeoutMS() { return connectionManager.getParams().getSoTimeout(); } /** * @param socketTimeoutMS the socketTimeoutMS to set */ public void setSocketTimeoutMS(int socketTimeoutMS) { connectionManager.getParams().setSoTimeout(socketTimeoutMS); } public String getRequestPrefix() { return requestPrefix; } public void setRequestPrefix(String requestPrefix) { this.requestPrefix = requestPrefix; } public HttpClient getHttpClient() { return http; } }