Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.solr.client.solrj.impl; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.MalformedURLException; import java.net.URL; import java.util.*; import java.util.zip.GZIPInputStream; import java.util.zip.InflaterInputStream; import org.apache.commons.httpclient.*; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.methods.InputStreamRequestEntity; import org.apache.commons.httpclient.methods.PostMethod; import org.apache.commons.httpclient.methods.RequestEntity; import org.apache.commons.httpclient.methods.multipart.MultipartRequestEntity; import org.apache.commons.httpclient.methods.multipart.FilePart; import org.apache.commons.httpclient.methods.multipart.Part; import org.apache.commons.httpclient.methods.multipart.PartSource; import org.apache.commons.httpclient.methods.multipart.StringPart; import org.apache.commons.httpclient.params.HttpMethodParams; import org.apache.solr.client.solrj.ResponseParser; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.response.UpdateResponse; import org.apache.solr.client.solrj.request.RequestWriter; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.util.ClientUtils; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.params.*; import org.apache.solr.common.util.ContentStream; import org.apache.solr.common.util.NamedList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * The {@link CommonsHttpSolrServer} uses the Apache Commons HTTP Client to connect to solr. * <pre class="prettyprint" >SolrServer server = new CommonsHttpSolrServer( url );</pre> * * @version $Id: CommonsHttpSolrServer.java 1170589 2011-09-14 13:03:51Z janhoy $ * @since solr 1.3 */ public class CommonsHttpSolrServer extends SolrServer { private static final long serialVersionUID = 1L; /** * User-Agent String as identified by the HTTP request by the {@link * org.apache.commons.httpclient.HttpClient HttpClient} to the Solr * server from the client. */ public static final String AGENT = "Solr[" + CommonsHttpSolrServer.class.getName() + "] 1.0"; private static Logger log = LoggerFactory.getLogger(CommonsHttpSolrServer.class); /** * The URL of the Solr server. */ protected String _baseURL; /** * Default value: null / empty. <p/> * Parameters that are added to every request regardless. This may be a place to add * something like an authentication token. */ protected ModifiableSolrParams _invariantParams; /** * Default response parser is BinaryResponseParser <p/> * This parser represents the default Response Parser chosen to * parse the response if the parser were not specified as part of * the request. * @see org.apache.solr.client.solrj.impl.BinaryResponseParser */ protected ResponseParser _parser; /** * The RequestWriter used to write all requests to Solr * @see org.apache.solr.client.solrj.request.RequestWriter */ protected RequestWriter requestWriter = new RequestWriter(); private final HttpClient _httpClient; /** * This defaults to false under the * assumption that if you are following a redirect to get to a Solr * installation, something is misconfigured somewhere. */ private boolean _followRedirects = false; /** * If compression is enabled, both gzip and deflate compression will * be accepted in the HTTP response. */ private boolean _allowCompression = false; /** * Maximum number of retries to attempt in the event of transient * errors. Default: 0 (no) retries. No more than 1 recommended. */ private int _maxRetries = 0; /** * Default value: <b> false </b> * <p> * If set to false, add the query parameters as URL-encoded parameters to the * POST request in a single part. If set to true, create a new part of a * multi-part request for each parameter. * * The reason for adding all parameters as parts of a multi-part request is * that this allows us to specify the charset -- standards for single-part * requests specify that non-ASCII characters should be URL-encoded, but don't * specify the charset of the characters to be URL-encoded (cf. * http://www.w3.org/TR/html401/interact/forms.html#form-content-type). * Therefore you have to rely on your servlet container doing the right thing * with single-part requests. */ private boolean useMultiPartPost; /** * @param solrServerUrl The URL of the Solr server. For * example, "<code>http://localhost:8983/solr/</code>" * if you are using the standard distribution Solr webapp * on your local machine. */ public CommonsHttpSolrServer(String solrServerUrl) throws MalformedURLException { this(new URL(solrServerUrl)); } /** * Talk to the Solr server via the given HttpClient. The connection manager * for the client should be a MultiThreadedHttpConnectionManager if this * client is being reused across SolrServer instances, or of multiple threads * will use this SolrServer. */ public CommonsHttpSolrServer(String solrServerUrl, HttpClient httpClient) throws MalformedURLException { this(new URL(solrServerUrl), httpClient, new BinaryResponseParser(), false); } public CommonsHttpSolrServer(String solrServerUrl, HttpClient httpClient, boolean useMultiPartPost) throws MalformedURLException { this(new URL(solrServerUrl), httpClient, new BinaryResponseParser(), useMultiPartPost); } public CommonsHttpSolrServer(String solrServerUrl, HttpClient httpClient, ResponseParser parser) throws MalformedURLException { this(new URL(solrServerUrl), httpClient, parser, false); } /** * @param baseURL The URL of the Solr server. For example, * "<code>http://localhost:8983/solr/</code>" if you are using the * standard distribution Solr webapp on your local machine. */ public CommonsHttpSolrServer(URL baseURL) { this(baseURL, null, new BinaryResponseParser(), false); } public CommonsHttpSolrServer(URL baseURL, HttpClient client) { this(baseURL, client, new BinaryResponseParser(), false); } /** * * @see #useMultiPartPost */ public CommonsHttpSolrServer(URL baseURL, HttpClient client, boolean useMultiPartPost) { this(baseURL, client, new BinaryResponseParser(), useMultiPartPost); } /** * @see #useMultiPartPost * @see #_parser */ public CommonsHttpSolrServer(URL baseURL, HttpClient client, ResponseParser parser, boolean useMultiPartPost) { _baseURL = baseURL.toExternalForm(); if (_baseURL.endsWith("/")) { _baseURL = _baseURL.substring(0, _baseURL.length() - 1); } if (_baseURL.indexOf('?') >= 0) { throw new RuntimeException( "Invalid base url for solrj. The base URL must not contain parameters: " + _baseURL); } if (client == null) { _httpClient = new HttpClient(new MultiThreadedHttpConnectionManager()); // prevent retries (note: this didn't work when set on mgr.. needed to be set on client) DefaultHttpMethodRetryHandler retryhandler = new DefaultHttpMethodRetryHandler(0, false); _httpClient.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryhandler); // set some better defaults if we created a new connection manager and client // increase the default connections this.setDefaultMaxConnectionsPerHost(32); // 2 this.setMaxTotalConnections(128); // 20 } else { _httpClient = client; } _parser = parser; this.useMultiPartPost = useMultiPartPost; } //------------------------------------------------------------------------ //------------------------------------------------------------------------ /** * Process the request. If {@link org.apache.solr.client.solrj.SolrRequest#getResponseParser()} is null, then use * {@link #getParser()} * @param request The {@link org.apache.solr.client.solrj.SolrRequest} to process * @return The {@link org.apache.solr.common.util.NamedList} result * @throws SolrServerException * @throws IOException * * @see #request(org.apache.solr.client.solrj.SolrRequest, org.apache.solr.client.solrj.ResponseParser) */ @Override public NamedList<Object> request(final SolrRequest request) throws SolrServerException, IOException { ResponseParser responseParser = request.getResponseParser(); if (responseParser == null) { responseParser = _parser; } return request(request, responseParser); } public NamedList<Object> request(final SolrRequest request, ResponseParser processor) throws SolrServerException, IOException { HttpMethod method = null; InputStream is = null; SolrParams params = request.getParams(); Collection<ContentStream> streams = requestWriter.getContentStreams(request); String path = requestWriter.getPath(request); if (path == null || !path.startsWith("/")) { path = "/select"; } ResponseParser parser = request.getResponseParser(); if (parser == null) { parser = _parser; } // The parser 'wt=' and 'version=' params are used instead of the original params ModifiableSolrParams wparams = new ModifiableSolrParams(); wparams.set(CommonParams.WT, parser.getWriterType()); wparams.set(CommonParams.VERSION, parser.getVersion()); if (params == null) { params = wparams; } else { params = new DefaultSolrParams(wparams, params); } if (_invariantParams != null) { params = new DefaultSolrParams(_invariantParams, params); } int tries = _maxRetries + 1; try { while (tries-- > 0) { // Note: since we aren't do intermittent time keeping // ourselves, the potential non-timeout latency could be as // much as tries-times (plus scheduling effects) the given // timeAllowed. try { if (SolrRequest.METHOD.GET == request.getMethod()) { if (streams != null) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!"); } method = new GetMethod(_baseURL + path + ClientUtils.toQueryString(params, false)); } else if (SolrRequest.METHOD.POST == request.getMethod()) { String url = _baseURL + path; boolean isMultipart = (streams != null && streams.size() > 1); if (streams == null || isMultipart) { PostMethod post = new PostMethod(url); post.getParams().setContentCharset("UTF-8"); if (!this.useMultiPartPost && !isMultipart) { post.addRequestHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"); } List<Part> parts = new LinkedList<Part>(); Iterator<String> iter = params.getParameterNamesIterator(); while (iter.hasNext()) { String p = iter.next(); String[] vals = params.getParams(p); if (vals != null) { for (String v : vals) { if (this.useMultiPartPost || isMultipart) { parts.add(new StringPart(p, v, "UTF-8")); } else { post.addParameter(p, v); } } } } if (isMultipart) { int i = 0; for (ContentStream content : streams) { final ContentStream c = content; String charSet = null; PartSource source = new PartSource() { public long getLength() { return c.getSize(); } public String getFileName() { return c.getName(); } public InputStream createInputStream() throws IOException { return c.getStream(); } }; parts.add(new FilePart(c.getName(), source, c.getContentType(), charSet)); } } if (parts.size() > 0) { post.setRequestEntity(new MultipartRequestEntity( parts.toArray(new Part[parts.size()]), post.getParams())); } method = post; } // It is has one stream, it is the post body, put the params in the URL else { String pstr = ClientUtils.toQueryString(params, false); PostMethod post = new PostMethod(url + pstr); // post.setRequestHeader("connection", "close"); // Single stream as body // Using a loop just to get the first one final ContentStream[] contentStream = new ContentStream[1]; for (ContentStream content : streams) { contentStream[0] = content; break; } if (contentStream[0] instanceof RequestWriter.LazyContentStream) { post.setRequestEntity(new RequestEntity() { public long getContentLength() { return -1; } public String getContentType() { return contentStream[0].getContentType(); } public boolean isRepeatable() { return false; } public void writeRequest(OutputStream outputStream) throws IOException { ((RequestWriter.LazyContentStream) contentStream[0]).writeTo(outputStream); } }); } else { is = contentStream[0].getStream(); post.setRequestEntity( new InputStreamRequestEntity(is, contentStream[0].getContentType())); } method = post; } } else { throw new SolrServerException("Unsupported method: " + request.getMethod()); } } catch (NoHttpResponseException r) { // This is generally safe to retry on method.releaseConnection(); method = null; if (is != null) { is.close(); } // If out of tries then just rethrow (as normal error). if ((tries < 1)) { throw r; } //log.warn( "Caught: " + r + ". Retrying..." ); } } } catch (IOException ex) { log.error("####request####", ex); throw new SolrServerException("error reading streams", ex); } method.setFollowRedirects(_followRedirects); method.addRequestHeader("User-Agent", AGENT); if (_allowCompression) { method.setRequestHeader(new Header("Accept-Encoding", "gzip,deflate")); } // method.setRequestHeader("connection", "close"); try { // Execute the method. //System.out.println( "EXECUTE:"+method.getURI() ); int statusCode = _httpClient.executeMethod(method); if (statusCode != HttpStatus.SC_OK) { StringBuilder msg = new StringBuilder(); msg.append(method.getStatusLine().getReasonPhrase()); msg.append("\n\n"); msg.append(method.getStatusText()); msg.append("\n\n"); msg.append("request: " + method.getURI()); throw new SolrException(statusCode, java.net.URLDecoder.decode(msg.toString(), "UTF-8")); } // Read the contents String charset = "UTF-8"; if (method instanceof HttpMethodBase) { charset = ((HttpMethodBase) method).getResponseCharSet(); } InputStream respBody = method.getResponseBodyAsStream(); // Jakarta Commons HTTPClient doesn't handle any // compression natively. Handle gzip or deflate // here if applicable. if (_allowCompression) { Header contentEncodingHeader = method.getResponseHeader("Content-Encoding"); if (contentEncodingHeader != null) { String contentEncoding = contentEncodingHeader.getValue(); if (contentEncoding.contains("gzip")) { //log.debug( "wrapping response in GZIPInputStream" ); respBody = new GZIPInputStream(respBody); } else if (contentEncoding.contains("deflate")) { //log.debug( "wrapping response in InflaterInputStream" ); respBody = new InflaterInputStream(respBody); } } else { Header contentTypeHeader = method.getResponseHeader("Content-Type"); if (contentTypeHeader != null) { String contentType = contentTypeHeader.getValue(); if (contentType != null) { if (contentType.startsWith("application/x-gzip-compressed")) { //log.debug( "wrapping response in GZIPInputStream" ); respBody = new GZIPInputStream(respBody); } else if (contentType.startsWith("application/x-deflate")) { //log.debug( "wrapping response in InflaterInputStream" ); respBody = new InflaterInputStream(respBody); } } } } } return processor.processResponse(respBody, charset); } catch (HttpException e) { throw new SolrServerException(e); } catch (IOException e) { throw new SolrServerException(e); } finally { method.releaseConnection(); if (is != null) { is.close(); } } } //------------------------------------------------------------------- //------------------------------------------------------------------- /** * Retrieve the default list of parameters are added to every request regardless. * * @see #_invariantParams */ public ModifiableSolrParams getInvariantParams() { return _invariantParams; } public String getBaseURL() { return _baseURL; } public void setBaseURL(String baseURL) { this._baseURL = baseURL; } public ResponseParser getParser() { return _parser; } /** * Note: This setter method is <b>not thread-safe</b>. * @param processor Default Response Parser chosen to parse the response if the parser were not specified as part of the request. * @see org.apache.solr.client.solrj.SolrRequest#getResponseParser() */ public void setParser(ResponseParser processor) { _parser = processor; } public HttpClient getHttpClient() { return _httpClient; } private HttpConnectionManager getConnectionManager() { return _httpClient.getHttpConnectionManager(); } /** set connectionTimeout on the underlying HttpConnectionManager * @param timeout Timeout in milliseconds **/ public void setConnectionTimeout(int timeout) { getConnectionManager().getParams().setConnectionTimeout(timeout); } /** set connectionManagerTimeout on the HttpClient. * @param timeout Timeout in milliseconds * @deprecated Use {@link #setConnectionManagerTimeout(long)} **/ @Deprecated public void setConnectionManagerTimeout(int timeout) { _httpClient.getParams().setConnectionManagerTimeout(timeout); } /** * Sets soTimeout (read timeout) on the underlying * HttpConnectionManager. This is desirable for queries, but * probably not for indexing. * * @param timeout Timeout in milliseconds */ public void setConnectionManagerTimeout(long timeout) { _httpClient.getParams().setConnectionManagerTimeout(timeout); } /** * Sets soTimeout (read timeout) on the underlying * HttpConnectionManager. This is desirable for queries, but * probably not for indexing. * @param timeout Timeout in milliseconds **/ public void setSoTimeout(int timeout) { getConnectionManager().getParams().setSoTimeout(timeout); } /** set maxConnectionsPerHost on the underlying HttpConnectionManager */ public void setDefaultMaxConnectionsPerHost(int connections) { getConnectionManager().getParams().setDefaultMaxConnectionsPerHost(connections); } /** set maxTotalConnection on the underlying HttpConnectionManager */ public void setMaxTotalConnections(int connections) { getConnectionManager().getParams().setMaxTotalConnections(connections); } /** * set followRedirects. * @see #_followRedirects */ public void setFollowRedirects(boolean followRedirects) { _followRedirects = followRedirects; } /** * set allowCompression. * @see #_allowCompression */ public void setAllowCompression(boolean allowCompression) { _allowCompression = allowCompression; } /** * set maximum number of retries to attempt in the event of * transient errors. * @param maxRetries No more than 1 recommended * @see #_maxRetries */ public void setMaxRetries(int maxRetries) { if (maxRetries > 1) { log.warn("CommonsHttpSolrServer: maximum Retries " + maxRetries + " > 1. Maximum recommended retries is 1."); } _maxRetries = maxRetries; } public void setRequestWriter(RequestWriter requestWriter) { this.requestWriter = requestWriter; } /** * Adds the documents supplied by the given iterator. * @param docIterator the iterator which returns SolrInputDocument instances * @return the response from the SolrServer */ public UpdateResponse add(Iterator<SolrInputDocument> docIterator) throws SolrServerException, IOException { return add(docIterator, -1); } /** * Adds the documents supplied by the given iterator, specifying max time before they become committed * @param docIterator the iterator which returns SolrInputDocument instances * @param commitWithinMs the time in milliseconds before a commit automatically is triggered * @return the response from the SolrServer */ public UpdateResponse add(Iterator<SolrInputDocument> docIterator, int commitWithinMs) throws SolrServerException, IOException { UpdateRequest req = new UpdateRequest(); req.setDocIterator(docIterator); req.setCommitWithin(commitWithinMs); return req.process(this); } /** * Adds the beans supplied by the given iterator * @param beanIterator the iterator which returns Beans * @return the response from the SolrServer */ public UpdateResponse addBeans(final Iterator<?> beanIterator) throws SolrServerException, IOException { return addBeans(beanIterator, -1); } /** * Adds the beans supplied by the given iterator, specifying max time before they become committed * @param commitWithinMs the time in milliseconds before a commit automatically is triggered * @param beanIterator the iterator which returns Beans * @return the response from the SolrServer */ public UpdateResponse addBeans(final Iterator<?> beanIterator, int commitWithinMs) throws SolrServerException, IOException { UpdateRequest req = new UpdateRequest(); req.setDocIterator(new Iterator<SolrInputDocument>() { public boolean hasNext() { return beanIterator.hasNext(); } public SolrInputDocument next() { Object o = beanIterator.next(); if (o == null) return null; return getBinder().toSolrInputDocument(o); } public void remove() { beanIterator.remove(); } }); req.setCommitWithin(commitWithinMs); return req.process(this); } }