Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.solr.handler.component; import org.apache.solr.handler.RequestHandlerBase; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.FacetParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.ShardParams; import org.apache.solr.common.SolrException; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.mdrill.FacetComponent; import org.apache.solr.request.mdrill.MdrillGroupBy; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer; import org.apache.solr.util.plugin.SolrCoreAware; import org.apache.solr.core.SolrCore; import org.apache.lucene.queryParser.ParseException; import org.apache.commons.httpclient.HttpClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; import java.util.concurrent.*; public class SearchHandler extends RequestHandlerBase implements SolrCoreAware { static final String INIT_COMPONENTS = "components"; static final String INIT_FIRST_COMPONENTS = "first-components"; static final String INIT_LAST_COMPONENTS = "last-components"; protected static Logger log = LoggerFactory.getLogger(SearchHandler.class); protected List<SearchComponent> components = null; protected List<String> getDefaultComponents() { ArrayList<String> names = new ArrayList<String>(6); names.add(QueryComponent.COMPONENT_NAME); names.add(FacetComponent.COMPONENT_NAME); return names; } @SuppressWarnings("unchecked") public void inform(SolrCore core) { if (components != null) { return; } List<String> list = getDefaultComponents(); components = new ArrayList<SearchComponent>(list.size()); for (String c : list) { SearchComponent comp = core.getSearchComponent(c); components.add(comp); log.info("Adding component:" + comp); } } public List<SearchComponent> getComponents() { return components; } @Override public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception, ParseException, InstantiationException, IllegalAccessException { ResponseBuilder rb = new ResponseBuilder(); rb.req = req; rb.rsp = rsp; rb.components = components; rb.setDebug(false);//??debug for (SearchComponent c : components) { c.prepare(rb); } if (rb.shards == null) { for (SearchComponent c : components) { c.process(rb); } } else { //merger server int depth = req.getParams().getInt("__higo_ms_depth__", 0); HttpCommComponent comm = new HttpCommComponent(depth); if (rb.outgoing == null) { rb.outgoing = new LinkedList<ShardRequest>(); } rb.finished = new ArrayList<ShardRequest>(); int nextStage = 0; do { rb.stage = nextStage; nextStage = ResponseBuilder.STAGE_DONE; // call all components for (SearchComponent c : components) { // the next stage is the minimum of what all components report nextStage = Math.min(nextStage, c.distributedProcess(rb)); } // check the outgoing queue and send requests while (rb.outgoing.size() > 0) { // submit all current request tasks at once while (rb.outgoing.size() > 0) { ShardRequest sreq = rb.outgoing.remove(0); sreq.actualShards = sreq.shards; if (sreq.actualShards == ShardRequest.ALL_SHARDS) { sreq.actualShards = rb.shards; } sreq.responses = new ArrayList<ShardResponse>(); for (int i = 0; i < sreq.actualShards.length; i++) { String shard = sreq.actualShards[i]; ModifiableSolrParams params = new ModifiableSolrParams(sreq.params); params.remove(ShardParams.SHARDS); // not a top-level request params.remove("indent"); params.remove(CommonParams.HEADER_ECHO_PARAMS); params.set(ShardParams.IS_SHARD, true); // a sub (shard) request params.set("higolockCount", rb.lockCnt); if (rb.issubshard) { params.set(ShardParams.SHARDS, rb.subShards[i]); params.set(FacetParams.IS_SUB_SHARDS, true); params.set(FacetParams.FACET_CROSS_OFFSET, 0); int maxlimit = MdrillGroupBy.MAX_CROSS_ROWS; params.set(FacetParams.FACET_CROSS_LIMIT, maxlimit); } String shardHandler = req.getParams().get(ShardParams.SHARDS_QT); if (shardHandler == null) { params.remove(CommonParams.QT); } else { params.set(CommonParams.QT, shardHandler); } comm.submit(sreq, shard, params, depth); } } while (rb.outgoing.size() == 0) { ShardResponse srsp = comm.takeCompletedOrError(); if (srsp == null) break; // no more requests to wait for if (srsp.getException() != null) { comm.cancelAll(); if (srsp.getException() instanceof SolrException) { throw (SolrException) srsp.getException(); } else { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, srsp.getException()); } } rb.finished.add(srsp.getShardRequest()); // let the components see the responses to the request for (SearchComponent c : components) { c.handleResponses(rb, srsp.getShardRequest()); } } } for (SearchComponent c : components) { c.finishStage(rb); } // we are done when the next stage is MAX_VALUE } while (nextStage != Integer.MAX_VALUE); } } //////////////////////// SolrInfoMBeans methods ////////////////////// @Override public String getDescription() { StringBuilder sb = new StringBuilder(); sb.append("Search using components: "); if (components != null) { for (SearchComponent c : components) { sb.append(c.getName()); sb.append(","); } } return sb.toString(); } @Override public String getVersion() { return "$Revision: 1052938 $"; } @Override public String getSourceId() { return "$Id: SearchHandler.java 1052938 2010-12-26 20:21:48Z rmuir $"; } @Override public String getSource() { return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene_solr_3_5/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java $"; } } class HttpCommComponent { CompletionService<ShardResponse> completionService = null; Set<Future<ShardResponse>> pending = new HashSet<Future<ShardResponse>>(); HttpCommComponent(int depth) { this.completionService = MergerServerThreads.create(depth); } public static class SimpleSolrResponse extends SolrResponse { long elapsedTime; NamedList<Object> nl; @Override public long getElapsedTime() { return elapsedTime; } @Override public NamedList<Object> getResponse() { return nl; } @Override public void setResponse(NamedList<Object> rsp) { nl = rsp; } } private static void setupParams(final ModifiableSolrParams params, String[] shardparams) { if (shardparams.length > 1) { params.set(CommonParams.PARTION, shardparams[1]); } if (shardparams.length > 2) { params.set("hadoop.merger.hashindex", shardparams[2]); } else { params.remove("hadoop.merger.hashindex"); } params.remove(CommonParams.WT); // use default (currently javabin) params.remove(CommonParams.VERSION); } private static HttpClient makeHttpClient() { HttpClient cli = new HttpClient(); cli.getHttpConnectionManager().getParams().setConnectionTimeout(1000 * 60 * 100); cli.getHttpConnectionManager().getParams().setSoTimeout(1000 * 60 * 100); cli.getParams().setConnectionManagerTimeout(1000 * 60 * 100); return cli; } void submit(final ShardRequest sreq, final String shard, final ModifiableSolrParams params, final int depth) { // ____higo_ms_depth__ Callable<ShardResponse> task = new Callable<ShardResponse>() { public ShardResponse call() throws Exception { ShardResponse srsp = new ShardResponse(); srsp.setShardRequest(sreq); srsp.setShard(shard); SimpleSolrResponse ssr = new SimpleSolrResponse(); srsp.setSolrResponse(ssr); long startTime = System.currentTimeMillis(); String[] shardparams = shard.split("@"); String url = "http://" + shardparams[0]; try { params.set("__higo_ms_depth__", (depth + 1)); HttpCommComponent.setupParams(params, shardparams); HttpClient cli = HttpCommComponent.makeHttpClient(); SolrServer server = new CommonsHttpSolrServer(url, cli); QueryRequest req = new QueryRequest(params); req.setMethod(SolrRequest.METHOD.POST); ssr.nl = server.request(req); cli.getHttpConnectionManager().closeIdleConnections(0); long t2 = System.currentTimeMillis(); SearchHandler.log.info("##HttpClient## timetaken=" + (t2 - startTime)); } catch (RuntimeException th) { SearchHandler.log.error(params.toString(), th); srsp.setException(new Exception(shard, th)); srsp.setResponseCode(-1); } catch (Throwable th) { SearchHandler.log.error(params.toString(), th); srsp.setException(new Exception(shard, th)); if (th instanceof SolrException) { srsp.setResponseCode(((SolrException) th).code()); } else { srsp.setResponseCode(-1); } } ssr.elapsedTime = System.currentTimeMillis() - startTime; return srsp; } }; pending.add(completionService.submit(task)); } /** returns a ShardResponse of the last response correlated with a ShardRequest */ ShardResponse take() { while (pending.size() > 0) { try { Future<ShardResponse> future = completionService.take(); pending.remove(future); ShardResponse rsp = future.get(); rsp.getShardRequest().responses.add(rsp); if (rsp.getShardRequest().responses.size() == rsp.getShardRequest().actualShards.length) { return rsp; } } catch (InterruptedException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); } catch (ExecutionException e) { // should be impossible... the problem with catching the exception // at this level is we don't know what ShardRequest it applied to throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Impossible Exception", e); } } return null; } /** returns a ShardResponse of the last response correlated with a ShardRequest, * or immediately returns a ShardResponse if there was an error detected */ ShardResponse takeCompletedOrError() { while (pending.size() > 0) { try { Future<ShardResponse> future = completionService.take(); pending.remove(future); ShardResponse rsp = future.get(); if (rsp.getException() != null) return rsp; // if exception, return immediately // add response to the response list... we do this after the take() and // not after the completion of "call" so we know when the last response // for a request was received. Otherwise we might return the same // request more than once. rsp.getShardRequest().responses.add(rsp); if (rsp.getShardRequest().responses.size() == rsp.getShardRequest().actualShards.length) { return rsp; } } catch (InterruptedException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); } catch (ExecutionException e) { // should be impossible... the problem with catching the exception // at this level is we don't know what ShardRequest it applied to throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Impossible Exception", e); } } return null; } void cancelAll() { for (Future<ShardResponse> future : pending) { future.cancel(true); } } }