org.apache.solr.servlet.HttpSolrCall.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.solr.servlet.HttpSolrCall.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.solr.servlet;

import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.lang.invoke.MethodHandles;
import java.security.Principal;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.TimeUnit;

import org.apache.commons.io.IOUtils;
import org.apache.commons.io.input.CloseShieldInputStream;
import org.apache.commons.io.output.CloseShieldOutputStream;
import org.apache.commons.lang.StringUtils;
import org.apache.http.Header;
import org.apache.http.HeaderIterator;
import org.apache.http.HttpEntity;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.entity.InputStreamEntity;
import org.apache.solr.api.ApiBag;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.Aliases;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
import org.apache.solr.common.util.ValidatingJsonMap;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.ContentStreamHandlerBase;
import org.apache.solr.logging.MDCLoggingContext;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryRequestBase;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.response.QueryResponseWriter;
import org.apache.solr.response.QueryResponseWriterUtil;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.security.AuthenticationPlugin;
import org.apache.solr.security.AuthorizationContext;
import org.apache.solr.security.AuthorizationContext.CollectionRequest;
import org.apache.solr.security.AuthorizationContext.RequestType;
import org.apache.solr.security.AuthorizationResponse;
import org.apache.solr.security.PKIAuthenticationPlugin;
import org.apache.solr.servlet.SolrDispatchFilter.Action;
import org.apache.solr.servlet.cache.HttpCacheHeaderUtil;
import org.apache.solr.servlet.cache.Method;
import org.apache.solr.update.processor.DistributingUpdateProcessorFactory;
import org.apache.solr.util.CommandOperation;
import org.apache.solr.util.JsonSchemaValidator;
import org.apache.solr.util.RTimerTree;
import org.apache.solr.util.TimeOut;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.NODE_NAME_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETE;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.RELOAD;
import static org.apache.solr.common.params.CommonParams.NAME;
import static org.apache.solr.common.params.CoreAdminParams.ACTION;
import static org.apache.solr.handler.admin.CollectionsHandler.SYSTEM_COLL;
import static org.apache.solr.servlet.SolrDispatchFilter.Action.ADMIN;
import static org.apache.solr.servlet.SolrDispatchFilter.Action.FORWARD;
import static org.apache.solr.servlet.SolrDispatchFilter.Action.PASSTHROUGH;
import static org.apache.solr.servlet.SolrDispatchFilter.Action.PROCESS;
import static org.apache.solr.servlet.SolrDispatchFilter.Action.REMOTEQUERY;
import static org.apache.solr.servlet.SolrDispatchFilter.Action.RETRY;
import static org.apache.solr.servlet.SolrDispatchFilter.Action.RETURN;

/**
 * This class represents a call made to Solr
 **/
public class HttpSolrCall {
    private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());

    static final Random random;
    static {
        // We try to make things reproducible in the context of our tests by initializing the random instance
        // based on the current seed
        String seed = System.getProperty("tests.seed");
        if (seed == null) {
            random = new Random();
        } else {
            random = new Random(seed.hashCode());
        }
    }

    protected final SolrDispatchFilter solrDispatchFilter;
    protected final CoreContainer cores;
    protected final HttpServletRequest req;
    protected final HttpServletResponse response;
    protected final boolean retry;
    protected SolrCore core = null;
    protected SolrQueryRequest solrReq = null;
    protected SolrRequestHandler handler = null;
    protected final SolrParams queryParams;
    protected String path;
    protected Action action;
    protected String coreUrl;
    protected SolrConfig config;
    protected Map<String, Integer> invalidStates;
    protected boolean usingAliases = false;

    //The states of client that is invalid in this request
    protected Aliases aliases = null;
    protected String corename = "";
    protected String origCorename = null;

    public RequestType getRequestType() {
        return requestType;
    }

    protected RequestType requestType;

    public List<String> getCollectionsList() {
        return collectionsList;
    }

    protected List<String> collectionsList;

    public HttpSolrCall(SolrDispatchFilter solrDispatchFilter, CoreContainer cores, HttpServletRequest request,
            HttpServletResponse response, boolean retry) {
        this.solrDispatchFilter = solrDispatchFilter;
        this.cores = cores;
        this.req = request;
        this.response = response;
        this.retry = retry;
        this.requestType = RequestType.UNKNOWN;
        queryParams = SolrRequestParsers.parseQueryString(req.getQueryString());
        // set a request timer which can be reused by requests if needed
        req.setAttribute(SolrRequestParsers.REQUEST_TIMER_SERVLET_ATTRIBUTE, new RTimerTree());
        // put the core container in request attribute
        req.setAttribute("org.apache.solr.CoreContainer", cores);
        path = req.getServletPath();
        if (req.getPathInfo() != null) {
            // this lets you handle /update/commit when /update is a servlet
            path += req.getPathInfo();
        }
        req.setAttribute(HttpSolrCall.class.getName(), this);
    }

    public String getPath() {
        return path;
    }

    public HttpServletRequest getReq() {
        return req;
    }

    public SolrCore getCore() {
        return core;
    }

    public SolrParams getQueryParams() {
        return queryParams;
    }

    protected void init() throws Exception {
        // check for management path
        String alternate = cores.getManagementPath();
        if (alternate != null && path.startsWith(alternate)) {
            path = path.substring(0, alternate.length());
        }
        // unused feature ?
        int idx = path.indexOf(':');
        if (idx > 0) {
            // save the portion after the ':' for a 'handler' path parameter
            path = path.substring(0, idx);
        }

        boolean usingAliases = false;

        // Check for container handlers
        handler = cores.getRequestHandler(path);
        if (handler != null) {
            solrReq = SolrRequestParsers.DEFAULT.parse(null, path, req);
            solrReq.getContext().put(CoreContainer.class.getName(), cores);
            requestType = RequestType.ADMIN;
            action = ADMIN;
            return;
        } else {
            //otherwise, we should find a core from the path
            idx = path.indexOf("/", 1);
            if (idx > 1) {
                // try to get the corename as a request parameter first
                corename = path.substring(1, idx);

                // look at aliases
                if (cores.isZooKeeperAware()) {
                    origCorename = corename;
                    ZkStateReader reader = cores.getZkController().getZkStateReader();
                    aliases = reader.getAliases();
                    if (aliases != null && aliases.collectionAliasSize() > 0) {
                        usingAliases = true;
                        String alias = aliases.getCollectionAlias(corename);
                        if (alias != null) {
                            collectionsList = StrUtils.splitSmart(alias, ",", true);
                            corename = collectionsList.get(0);
                        }
                    }
                }

                core = cores.getCore(corename);
                if (core != null) {
                    path = path.substring(idx);
                } else if (cores.isCoreLoading(corename)) { // extra mem barriers, so don't look at this before trying to get core
                    throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "SolrCore is loading");
                } else {
                    // the core may have just finished loading
                    core = cores.getCore(corename);
                    if (core != null) {
                        path = path.substring(idx);
                    }
                }
            }
            if (core == null) {
                if (!cores.isZooKeeperAware()) {
                    core = cores.getCore("");
                }
            }
        }

        if (core == null && cores.isZooKeeperAware()) {
            // we couldn't find the core - lets make sure a collection was not specified instead
            boolean isPreferLeader = false;
            if (path.endsWith("/update") || path.contains("/update/")) {
                isPreferLeader = true;
            }
            core = getCoreByCollection(corename, isPreferLeader);
            if (core != null) {
                // we found a core, update the path
                path = path.substring(idx);
                if (collectionsList == null)
                    collectionsList = new ArrayList<>();
                collectionsList.add(corename);
            }

            // if we couldn't find it locally, look on other nodes
            extractRemotePath(corename, origCorename, idx);
            if (action != null)
                return;
            //core is not available locally or remotely
            autoCreateSystemColl(corename);
            if (action != null)
                return;
        }

        // With a valid core...
        if (core != null) {
            MDCLoggingContext.setCore(core);
            config = core.getSolrConfig();
            // get or create/cache the parser for the core
            SolrRequestParsers parser = config.getRequestParsers();

            // Determine the handler from the url path if not set
            // (we might already have selected the cores handler)
            extractHandlerFromURLPath(parser);
            if (action != null)
                return;

            // With a valid handler and a valid core...
            if (handler != null) {
                // if not a /select, create the request
                if (solrReq == null) {
                    solrReq = parser.parse(core, path, req);
                }

                if (usingAliases) {
                    processAliases(aliases, collectionsList);
                }

                action = PROCESS;
                return; // we are done with a valid handler
            }
        }
        log.debug("no handler or core retrieved for " + path + ", follow through...");

        action = PASSTHROUGH;
    }

    protected void autoCreateSystemColl(String corename) throws Exception {
        if (core == null && SYSTEM_COLL.equals(corename) && "POST".equals(req.getMethod())
                && !cores.getZkController().getClusterState().hasCollection(SYSTEM_COLL)) {
            log.info("Going to auto-create .system collection");
            SolrQueryResponse rsp = new SolrQueryResponse();
            String repFactor = String
                    .valueOf(Math.min(3, cores.getZkController().getClusterState().getLiveNodes().size()));
            cores.getCollectionsHandler().handleRequestBody(
                    new LocalSolrQueryRequest(null, new ModifiableSolrParams().add(ACTION, CREATE.toString())
                            .add(NAME, SYSTEM_COLL).add(REPLICATION_FACTOR, repFactor)),
                    rsp);
            if (rsp.getValues().get("success") == null) {
                throw new SolrException(ErrorCode.SERVER_ERROR,
                        "Could not auto-create .system collection: " + Utils.toJSONString(rsp.getValues()));
            }
            TimeOut timeOut = new TimeOut(3, TimeUnit.SECONDS);
            for (;;) {
                if (cores.getZkController().getClusterState().getCollectionOrNull(SYSTEM_COLL) != null) {
                    break;
                } else {
                    if (timeOut.hasTimedOut()) {
                        throw new SolrException(ErrorCode.SERVER_ERROR,
                                "Could not find .system collection even after 3 seconds");
                    }
                    Thread.sleep(50);
                }
            }

            action = RETRY;
        }
    }

    protected String lookupAliases(String collName) {
        ZkStateReader reader = cores.getZkController().getZkStateReader();
        aliases = reader.getAliases();
        if (aliases != null && aliases.collectionAliasSize() > 0) {
            usingAliases = true;
            String alias = aliases.getCollectionAlias(collName);
            if (alias != null) {
                collectionsList = StrUtils.splitSmart(alias, ",", true);
                return collectionsList.get(0);
            }
        }
        return null;
    }

    /**
     * Extract handler from the URL path if not set.
     * This returns true if the action is set.
     * 
     */
    protected void extractHandlerFromURLPath(SolrRequestParsers parser) throws Exception {
        if (handler == null && path.length() > 1) { // don't match "" or "/" as valid path
            handler = core.getRequestHandler(path);

            if (handler == null) {
                //may be a restlet path
                // Handle /schema/* paths via Restlet
                if (path.equals("/schema") || path.startsWith("/schema/")) {
                    solrReq = parser.parse(core, path, req);
                    SolrRequestInfo.setRequestInfo(new SolrRequestInfo(solrReq, new SolrQueryResponse()));
                    if (path.equals(req.getServletPath())) {
                        // avoid endless loop - pass through to Restlet via webapp
                        action = PASSTHROUGH;
                        return;
                    } else {
                        // forward rewritten URI (without path prefix and core/collection name) to Restlet
                        action = FORWARD;
                        return;
                    }
                }

            }
            // no handler yet but allowed to handle select; let's check

            if (handler == null && parser.isHandleSelect()) {
                if ("/select".equals(path) || "/select/".equals(path)) {
                    solrReq = parser.parse(core, path, req);
                    invalidStates = checkStateIsValid(solrReq.getParams().get(CloudSolrClient.STATE_VERSION));
                    String qt = solrReq.getParams().get(CommonParams.QT);
                    handler = core.getRequestHandler(qt);
                    if (handler == null) {
                        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "unknown handler: " + qt);
                    }
                    if (qt != null && qt.startsWith("/") && (handler instanceof ContentStreamHandlerBase)) {
                        //For security reasons it's a bad idea to allow a leading '/', ex: /select?qt=/update see SOLR-3161
                        //There was no restriction from Solr 1.4 thru 3.5 and it's not supported for update handlers.
                        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
                                "Invalid Request Handler ('qt').  Do not use /select to access: " + qt);
                    }
                }
            }
        }
    }

    protected void extractRemotePath(String corename, String origCorename, int idx)
            throws UnsupportedEncodingException, KeeperException, InterruptedException {
        if (core == null && idx > 0) {
            coreUrl = getRemotCoreUrl(corename, origCorename);
            // don't proxy for internal update requests
            invalidStates = checkStateIsValid(queryParams.get(CloudSolrClient.STATE_VERSION));
            if (coreUrl != null
                    && queryParams.get(DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM) == null) {
                path = path.substring(idx);
                if (invalidStates != null) {
                    //it does not make sense to send the request to a remote node
                    throw new SolrException(SolrException.ErrorCode.INVALID_STATE,
                            new String(Utils.toJSON(invalidStates), org.apache.lucene.util.IOUtils.UTF_8));
                }
                action = REMOTEQUERY;
            } else {
                if (!retry) {
                    // we couldn't find a core to work with, try reloading aliases
                    // TODO: it would be nice if admin ui elements skipped this...
                    ZkStateReader reader = cores.getZkController().getZkStateReader();
                    reader.updateAliases();
                    action = RETRY;
                }
            }
        }
    }

    /**
     * This method processes the request.
     */
    public Action call() throws IOException {
        MDCLoggingContext.reset();
        MDCLoggingContext.setNode(cores);

        if (cores == null) {
            sendError(503, "Server is shutting down or failed to initialize");
            return RETURN;
        }

        if (solrDispatchFilter.abortErrorMessage != null) {
            sendError(500, solrDispatchFilter.abortErrorMessage);
            return RETURN;
        }

        try {
            init();
            /* Authorize the request if
             1. Authorization is enabled, and
             2. The requested resource is not a known static file
              */
            if (cores.getAuthorizationPlugin() != null && shouldAuthorize()) {
                AuthorizationContext context = getAuthCtx();
                log.debug("AuthorizationContext : {}", context);
                AuthorizationResponse authResponse = cores.getAuthorizationPlugin().authorize(context);
                if (authResponse.statusCode == AuthorizationResponse.PROMPT.statusCode) {
                    Map<String, String> headers = (Map) getReq().getAttribute(AuthenticationPlugin.class.getName());
                    if (headers != null) {
                        for (Map.Entry<String, String> e : headers.entrySet())
                            response.setHeader(e.getKey(), e.getValue());
                    }
                    log.debug("USER_REQUIRED " + req.getHeader("Authorization") + " " + req.getUserPrincipal());
                }
                if (!(authResponse.statusCode == HttpStatus.SC_ACCEPTED)
                        && !(authResponse.statusCode == HttpStatus.SC_OK)) {
                    log.info("USER_REQUIRED auth header {} context : {} ", req.getHeader("Authorization"), context);
                    sendError(authResponse.statusCode,
                            "Unauthorized request, Response code: " + authResponse.statusCode);
                    return RETURN;
                }
            }

            HttpServletResponse resp = response;
            switch (action) {
            case ADMIN:
                handleAdminRequest();
                return RETURN;
            case REMOTEQUERY:
                remoteQuery(coreUrl + path, resp);
                return RETURN;
            case PROCESS:
                final Method reqMethod = Method.getMethod(req.getMethod());
                HttpCacheHeaderUtil.setCacheControlHeader(config, resp, reqMethod);
                // unless we have been explicitly told not to, do cache validation
                // if we fail cache validation, execute the query
                if (config.getHttpCachingConfig().isNever304()
                        || !HttpCacheHeaderUtil.doCacheHeaderValidation(solrReq, req, reqMethod, resp)) {
                    SolrQueryResponse solrRsp = new SolrQueryResponse();
                    /* even for HEAD requests, we need to execute the handler to
                     * ensure we don't get an error (and to make sure the correct
                     * QueryResponseWriter is selected and we get the correct
                     * Content-Type)
                     */
                    SolrRequestInfo.setRequestInfo(new SolrRequestInfo(solrReq, solrRsp));
                    execute(solrRsp);
                    HttpCacheHeaderUtil.checkHttpCachingVeto(solrRsp, resp, reqMethod);
                    Iterator<Map.Entry<String, String>> headers = solrRsp.httpHeaders();
                    while (headers.hasNext()) {
                        Map.Entry<String, String> entry = headers.next();
                        resp.addHeader(entry.getKey(), entry.getValue());
                    }
                    QueryResponseWriter responseWriter = getResponseWriter();
                    if (invalidStates != null)
                        solrReq.getContext().put(CloudSolrClient.STATE_VERSION, invalidStates);
                    writeResponse(solrRsp, responseWriter, reqMethod);
                }
                return RETURN;
            default:
                return action;
            }
        } catch (Throwable ex) {
            sendError(ex);
            // walk the the entire cause chain to search for an Error
            Throwable t = ex;
            while (t != null) {
                if (t instanceof Error) {
                    if (t != ex) {
                        log.error(
                                "An Error was wrapped in another exception - please report complete stacktrace on SOLR-6161",
                                ex);
                    }
                    throw (Error) t;
                }
                t = t.getCause();
            }
            return RETURN;
        } finally {
            MDCLoggingContext.clear();
        }

    }

    private boolean shouldAuthorize() {
        if (PKIAuthenticationPlugin.PATH.equals(path))
            return false;
        //admin/info/key is the path where public key is exposed . it is always unsecured
        if (cores.getPkiAuthenticationPlugin() != null && req.getUserPrincipal() != null) {
            boolean b = cores.getPkiAuthenticationPlugin().needsAuthorization(req);
            log.debug("PkiAuthenticationPlugin says authorization required : {} ", b);
            return b;
        }
        return true;
    }

    void destroy() {
        try {
            if (solrReq != null) {
                log.debug("Closing out SolrRequest: {}", solrReq);
                solrReq.close();
            }
        } finally {
            try {
                if (core != null)
                    core.close();
            } finally {
                SolrRequestInfo.clearRequestInfo();
            }
            AuthenticationPlugin authcPlugin = cores.getAuthenticationPlugin();
            if (authcPlugin != null)
                authcPlugin.closeRequest();
        }
    }

    private void remoteQuery(String coreUrl, HttpServletResponse resp) throws IOException {
        HttpRequestBase method = null;
        HttpEntity httpEntity = null;
        try {
            String urlstr = coreUrl + queryParams.toQueryString();

            boolean isPostOrPutRequest = "POST".equals(req.getMethod()) || "PUT".equals(req.getMethod());
            if ("GET".equals(req.getMethod())) {
                method = new HttpGet(urlstr);
            } else if ("HEAD".equals(req.getMethod())) {
                method = new HttpHead(urlstr);
            } else if (isPostOrPutRequest) {
                HttpEntityEnclosingRequestBase entityRequest = "POST".equals(req.getMethod()) ? new HttpPost(urlstr)
                        : new HttpPut(urlstr);
                InputStream in = new CloseShieldInputStream(req.getInputStream()); // Prevent close of container streams
                HttpEntity entity = new InputStreamEntity(in, req.getContentLength());
                entityRequest.setEntity(entity);
                method = entityRequest;
            } else if ("DELETE".equals(req.getMethod())) {
                method = new HttpDelete(urlstr);
            } else {
                throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
                        "Unexpected method type: " + req.getMethod());
            }

            for (Enumeration<String> e = req.getHeaderNames(); e.hasMoreElements();) {
                String headerName = e.nextElement();
                if (!"host".equalsIgnoreCase(headerName) && !"authorization".equalsIgnoreCase(headerName)
                        && !"accept".equalsIgnoreCase(headerName)) {
                    method.addHeader(headerName, req.getHeader(headerName));
                }
            }
            // These headers not supported for HttpEntityEnclosingRequests
            if (method instanceof HttpEntityEnclosingRequest) {
                method.removeHeaders(TRANSFER_ENCODING_HEADER);
                method.removeHeaders(CONTENT_LENGTH_HEADER);
            }

            final HttpResponse response = solrDispatchFilter.httpClient.execute(method,
                    HttpClientUtil.createNewHttpClientRequestContext());
            int httpStatus = response.getStatusLine().getStatusCode();
            httpEntity = response.getEntity();

            resp.setStatus(httpStatus);
            for (HeaderIterator responseHeaders = response.headerIterator(); responseHeaders.hasNext();) {
                Header header = responseHeaders.nextHeader();

                // We pull out these two headers below because they can cause chunked
                // encoding issues with Tomcat
                if (header != null && !header.getName().equalsIgnoreCase(TRANSFER_ENCODING_HEADER)
                        && !header.getName().equalsIgnoreCase(CONNECTION_HEADER)) {
                    resp.addHeader(header.getName(), header.getValue());
                }
            }

            if (httpEntity != null) {
                if (httpEntity.getContentEncoding() != null)
                    resp.setCharacterEncoding(httpEntity.getContentEncoding().getValue());
                if (httpEntity.getContentType() != null)
                    resp.setContentType(httpEntity.getContentType().getValue());

                InputStream is = httpEntity.getContent();
                OutputStream os = resp.getOutputStream();

                IOUtils.copyLarge(is, os);
            }

        } catch (IOException e) {
            sendError(new SolrException(SolrException.ErrorCode.SERVER_ERROR,
                    "Error trying to proxy request for url: " + coreUrl, e));
        } finally {
            Utils.consumeFully(httpEntity);
        }

    }

    protected void sendError(Throwable ex) throws IOException {
        Exception exp = null;
        SolrCore localCore = null;
        try {
            SolrQueryResponse solrResp = new SolrQueryResponse();
            if (ex instanceof Exception) {
                solrResp.setException((Exception) ex);
            } else {
                solrResp.setException(new RuntimeException(ex));
            }
            localCore = core;
            if (solrReq == null) {
                final SolrParams solrParams;
                if (req != null) {
                    // use GET parameters if available:
                    solrParams = SolrRequestParsers.parseQueryString(req.getQueryString());
                } else {
                    // we have no params at all, use empty ones:
                    solrParams = new MapSolrParams(Collections.<String, String>emptyMap());
                }
                solrReq = new SolrQueryRequestBase(core, solrParams) {
                };
            }
            QueryResponseWriter writer = core.getQueryResponseWriter(solrReq);
            writeResponse(solrResp, writer, Method.GET);
        } catch (Exception e) { // This error really does not matter
            exp = e;
        } finally {
            try {
                if (exp != null) {
                    SimpleOrderedMap info = new SimpleOrderedMap();
                    int code = ResponseUtils.getErrorInfo(ex, info, log);
                    sendError(code, info.toString());
                }
            } finally {
                if (core == null && localCore != null) {
                    localCore.close();
                }
            }
        }
    }

    protected void sendError(int code, String message) throws IOException {
        try {
            response.sendError(code, message);
        } catch (EOFException e) {
            log.info("Unable to write error response, client closed connection or we are shutting down", e);
        }
    }

    protected void execute(SolrQueryResponse rsp) {
        // a custom filter could add more stuff to the request before passing it on.
        // for example: sreq.getContext().put( "HttpServletRequest", req );
        // used for logging query stats in SolrCore.execute()
        solrReq.getContext().put("webapp", req.getContextPath());
        solrReq.getCore().execute(handler, solrReq, rsp);
    }

    private void handleAdminRequest() throws IOException {
        SolrQueryResponse solrResp = new SolrQueryResponse();
        SolrCore.preDecorateResponse(solrReq, solrResp);
        handleAdmin(solrResp);
        SolrCore.postDecorateResponse(handler, solrReq, solrResp);
        if (log.isInfoEnabled() && solrResp.getToLog().size() > 0) {
            log.info(solrResp.getToLogAsString("[admin]"));
        }
        QueryResponseWriter respWriter = SolrCore.DEFAULT_RESPONSE_WRITERS
                .get(solrReq.getParams().get(CommonParams.WT));
        if (respWriter == null)
            respWriter = getResponseWriter();
        writeResponse(solrResp, respWriter, Method.getMethod(req.getMethod()));
    }

    protected QueryResponseWriter getResponseWriter() {
        if (core != null)
            return core.getQueryResponseWriter(solrReq);
        QueryResponseWriter respWriter = SolrCore.DEFAULT_RESPONSE_WRITERS
                .get(solrReq.getParams().get(CommonParams.WT));
        if (respWriter == null)
            respWriter = SolrCore.DEFAULT_RESPONSE_WRITERS.get("standard");
        return respWriter;

    }

    protected void handleAdmin(SolrQueryResponse solrResp) {
        handler.handleRequest(solrReq, solrResp);
    }

    protected void processAliases(Aliases aliases, List<String> collectionsList) {
        String collection = solrReq.getParams().get(COLLECTION_PROP);
        if (collection != null) {
            collectionsList = StrUtils.splitSmart(collection, ",", true);
        }
        if (collectionsList != null) {
            Set<String> newCollectionsList = new HashSet<>(collectionsList.size());
            for (String col : collectionsList) {
                String al = aliases.getCollectionAlias(col);
                if (al != null) {
                    List<String> aliasList = StrUtils.splitSmart(al, ",", true);
                    newCollectionsList.addAll(aliasList);
                } else {
                    newCollectionsList.add(col);
                }
            }
            if (newCollectionsList.size() > 0) {
                StringBuilder collectionString = new StringBuilder();
                Iterator<String> it = newCollectionsList.iterator();
                int sz = newCollectionsList.size();
                for (int i = 0; i < sz; i++) {
                    collectionString.append(it.next());
                    if (i < newCollectionsList.size() - 1) {
                        collectionString.append(",");
                    }
                }
                ModifiableSolrParams params = new ModifiableSolrParams(solrReq.getParams());
                params.set(COLLECTION_PROP, collectionString.toString());
                solrReq.setParams(params);
            }
        }
    }

    private void writeResponse(SolrQueryResponse solrRsp, QueryResponseWriter responseWriter, Method reqMethod)
            throws IOException {
        try {
            Object invalidStates = solrReq.getContext().get(CloudSolrClient.STATE_VERSION);
            //This is the last item added to the response and the client would expect it that way.
            //If that assumption is changed , it would fail. This is done to avoid an O(n) scan on
            // the response for each request
            if (invalidStates != null)
                solrRsp.add(CloudSolrClient.STATE_VERSION, invalidStates);
            // Now write it out
            final String ct = responseWriter.getContentType(solrReq, solrRsp);
            // don't call setContentType on null
            if (null != ct)
                response.setContentType(ct);

            if (solrRsp.getException() != null) {
                NamedList info = new SimpleOrderedMap();
                int code = ResponseUtils.getErrorInfo(solrRsp.getException(), info, log);
                solrRsp.add("error", info);
                response.setStatus(code);
            }

            if (Method.HEAD != reqMethod) {
                OutputStream out = new CloseShieldOutputStream(response.getOutputStream()); // Prevent close of container streams, see SOLR-8933
                QueryResponseWriterUtil.writeQueryResponse(out, responseWriter, solrReq, solrRsp, ct);
            }
            //else http HEAD request, nothing to write out, waited this long just to get ContentType
        } catch (EOFException e) {
            log.info("Unable to write response, client closed connection or we are shutting down", e);
        }
    }

    private Map<String, Integer> checkStateIsValid(String stateVer) {
        Map<String, Integer> result = null;
        String[] pairs;
        if (stateVer != null && !stateVer.isEmpty() && cores.isZooKeeperAware()) {
            // many have multiple collections separated by |
            pairs = StringUtils.split(stateVer, '|');
            for (String pair : pairs) {
                String[] pcs = StringUtils.split(pair, ':');
                if (pcs.length == 2 && !pcs[0].isEmpty() && !pcs[1].isEmpty()) {
                    Integer status = cores.getZkController().getZkStateReader().compareStateVersions(pcs[0],
                            Integer.parseInt(pcs[1]));
                    if (status != null) {
                        if (result == null)
                            result = new HashMap<>();
                        result.put(pcs[0], status);
                    }
                }
            }
        }
        return result;
    }

    protected SolrCore getCoreByCollection(String collectionName, boolean isPreferLeader) {
        ZkStateReader zkStateReader = cores.getZkController().getZkStateReader();

        ClusterState clusterState = zkStateReader.getClusterState();
        DocCollection collection = clusterState.getCollectionOrNull(collectionName);
        if (collection == null) {
            return null;
        }

        Set<String> liveNodes = clusterState.getLiveNodes();

        if (isPreferLeader) {
            List<Replica> leaderReplicas = collection.getLeaderReplicas(cores.getZkController().getNodeName());
            SolrCore core = randomlyGetSolrCore(liveNodes, leaderReplicas);
            if (core != null)
                return core;
        }

        List<Replica> replicas = collection.getReplicas(cores.getZkController().getNodeName());
        return randomlyGetSolrCore(liveNodes, replicas);
    }

    private SolrCore randomlyGetSolrCore(Set<String> liveNodes, List<Replica> replicas) {
        if (replicas != null) {
            RandomIterator<Replica> it = new RandomIterator<>(random, replicas);
            while (it.hasNext()) {
                Replica replica = it.next();
                if (liveNodes.contains(replica.getNodeName()) && replica.getState() == Replica.State.ACTIVE) {
                    SolrCore core = checkProps(replica);
                    if (core != null)
                        return core;
                }
            }
        }
        return null;
    }

    private SolrCore checkProps(ZkNodeProps zkProps) {
        String corename;
        SolrCore core = null;
        if (cores.getZkController().getNodeName().equals(zkProps.getStr(NODE_NAME_PROP))) {
            corename = zkProps.getStr(CORE_NAME_PROP);
            core = cores.getCore(corename);
        }
        return core;
    }

    private void getSlicesForCollections(ClusterState clusterState, Collection<Slice> slices,
            boolean activeSlices) {
        if (activeSlices) {
            for (Map.Entry<String, DocCollection> entry : clusterState.getCollectionsMap().entrySet()) {
                final Collection<Slice> activeCollectionSlices = entry.getValue().getActiveSlices();
                if (activeCollectionSlices != null) {
                    slices.addAll(activeCollectionSlices);
                }
            }
        } else {
            for (Map.Entry<String, DocCollection> entry : clusterState.getCollectionsMap().entrySet()) {
                final Collection<Slice> collectionSlices = entry.getValue().getSlices();
                if (collectionSlices != null) {
                    slices.addAll(collectionSlices);
                }
            }
        }
    }

    private String getRemotCoreUrl(String collectionName, String origCorename) {
        ClusterState clusterState = cores.getZkController().getClusterState();
        Collection<Slice> slices = clusterState.getActiveSlices(collectionName);
        boolean byCoreName = false;

        if (slices == null) {
            slices = new ArrayList<>();
            // look by core name
            byCoreName = true;
            getSlicesForCollections(clusterState, slices, true);
            if (slices.isEmpty()) {
                getSlicesForCollections(clusterState, slices, false);
            }
        }

        if (slices.isEmpty()) {
            return null;
        }

        if (collectionsList == null)
            collectionsList = new ArrayList<>();

        collectionsList.add(collectionName);
        String coreUrl = getCoreUrl(collectionName, origCorename, clusterState, slices, byCoreName, true);

        if (coreUrl == null) {
            coreUrl = getCoreUrl(collectionName, origCorename, clusterState, slices, byCoreName, false);
        }

        return coreUrl;
    }

    private String getCoreUrl(String collectionName, String origCorename, ClusterState clusterState,
            Collection<Slice> slices, boolean byCoreName, boolean activeReplicas) {
        String coreUrl;
        Set<String> liveNodes = clusterState.getLiveNodes();
        List<Slice> randomizedSlices = new ArrayList<>(slices.size());
        randomizedSlices.addAll(slices);
        Collections.shuffle(randomizedSlices, random);

        for (Slice slice : randomizedSlices) {
            List<Replica> randomizedReplicas = new ArrayList<>();
            randomizedReplicas.addAll(slice.getReplicas());
            Collections.shuffle(randomizedReplicas, random);

            for (Replica replica : randomizedReplicas) {
                if (!activeReplicas || (liveNodes.contains(replica.getNodeName())
                        && replica.getState() == Replica.State.ACTIVE)) {

                    if (byCoreName && !collectionName.equals(replica.getStr(CORE_NAME_PROP))) {
                        // if it's by core name, make sure they match
                        continue;
                    }
                    if (replica.getStr(BASE_URL_PROP).equals(cores.getZkController().getBaseUrl())) {
                        // don't count a local core
                        continue;
                    }

                    if (origCorename != null) {
                        coreUrl = replica.getStr(BASE_URL_PROP) + "/" + origCorename;
                    } else {
                        coreUrl = replica.getCoreUrl();
                        if (coreUrl.endsWith("/")) {
                            coreUrl = coreUrl.substring(0, coreUrl.length() - 1);
                        }
                    }

                    return coreUrl;
                }
            }
        }
        return null;
    }

    protected Object _getHandler() {
        return handler;
    }

    private AuthorizationContext getAuthCtx() {

        String resource = getPath();

        SolrParams params = getQueryParams();
        final ArrayList<CollectionRequest> collectionRequests = new ArrayList<>();
        if (getCollectionsList() != null) {
            for (String collection : getCollectionsList()) {
                collectionRequests.add(new CollectionRequest(collection));
            }
        }

        // Extract collection name from the params in case of a Collection Admin request
        if (getPath().equals("/admin/collections")) {
            if (CREATE.isEqual(params.get("action")) || RELOAD.isEqual(params.get("action"))
                    || DELETE.isEqual(params.get("action")))
                collectionRequests.add(new CollectionRequest(params.get("name")));
            else if (params.get(COLLECTION_PROP) != null)
                collectionRequests.add(new CollectionRequest(params.get(COLLECTION_PROP)));
        }

        // Handle the case when it's a /select request and collections are specified as a param
        if (resource.equals("/select") && params.get("collection") != null) {
            collectionRequests.clear();
            for (String collection : params.get("collection").split(",")) {
                collectionRequests.add(new CollectionRequest(collection));
            }
        }

        // Populate the request type if the request is select or update
        if (requestType == RequestType.UNKNOWN) {
            if (resource.startsWith("/select") || resource.startsWith("/get"))
                requestType = RequestType.READ;
            if (resource.startsWith("/update"))
                requestType = RequestType.WRITE;
        }

        // There's no collection explicitly mentioned, let's try and extract it from the core if one exists for
        // the purpose of processing this request.
        if (getCore() != null && (getCollectionsList() == null || getCollectionsList().size() == 0)) {
            collectionRequests.add(new CollectionRequest(getCore().getCoreDescriptor().getCollectionName()));
        }

        if (getQueryParams().get(COLLECTION_PROP) != null)
            collectionRequests.add(new CollectionRequest(getQueryParams().get(COLLECTION_PROP)));

        return new AuthorizationContext() {
            @Override
            public SolrParams getParams() {
                return solrReq.getParams();
            }

            @Override
            public Principal getUserPrincipal() {
                return getReq().getUserPrincipal();
            }

            @Override
            public String getHttpHeader(String s) {
                return getReq().getHeader(s);
            }

            @Override
            public Enumeration getHeaderNames() {
                return getReq().getHeaderNames();
            }

            @Override
            public List<CollectionRequest> getCollectionRequests() {
                return collectionRequests;
            }

            @Override
            public RequestType getRequestType() {
                return requestType;
            }

            public String getResource() {
                return path;
            }

            @Override
            public String getHttpMethod() {
                return getReq().getMethod();
            }

            @Override
            public Object getHandler() {
                return _getHandler();
            }

            @Override
            public String toString() {
                StringBuilder response = new StringBuilder("userPrincipal: [").append(getUserPrincipal())
                        .append("]").append(" type: [").append(requestType.toString()).append("], collections: [");
                for (CollectionRequest collectionRequest : collectionRequests) {
                    response.append(collectionRequest.collectionName).append(", ");
                }
                if (collectionRequests.size() > 0)
                    response.delete(response.length() - 1, response.length());

                response.append("], Path: [").append(resource).append("]");
                response.append(" path : ").append(path).append(" params :").append(solrReq.getParams());
                return response.toString();
            }

            @Override
            public String getRemoteAddr() {
                return getReq().getRemoteAddr();
            }

            @Override
            public String getRemoteHost() {
                return getReq().getRemoteHost();
            }
        };

    }

    static final String CONNECTION_HEADER = "Connection";
    static final String TRANSFER_ENCODING_HEADER = "Transfer-Encoding";
    static final String CONTENT_LENGTH_HEADER = "Content-Length";
    List<CommandOperation> parsedCommands;

    public List<CommandOperation> getCommands(boolean validateInput) {
        if (parsedCommands == null) {
            Iterable<ContentStream> contentStreams = solrReq.getContentStreams();
            if (contentStreams == null)
                parsedCommands = Collections.EMPTY_LIST;
            else {
                for (ContentStream contentStream : contentStreams) {
                    try {
                        parsedCommands = ApiBag.getCommandOperations(contentStream.getReader(), getValidators(),
                                validateInput);
                    } catch (IOException e) {
                        throw new SolrException(ErrorCode.BAD_REQUEST, "Error reading commands");
                    }
                    break;
                }
            }
        }
        return CommandOperation.clone(parsedCommands);
    }

    protected ValidatingJsonMap getSpec() {
        return null;
    }

    protected Map<String, JsonSchemaValidator> getValidators() {
        return Collections.EMPTY_MAP;
    }

    /**
     * A faster method for randomly picking items when you do not need to
     * consume all items.
     */
    private static class RandomIterator<E> implements Iterator<E> {
        private Random rand;
        private ArrayList<E> elements;
        private int size;

        public RandomIterator(Random rand, Collection<E> elements) {
            this.rand = rand;
            this.elements = new ArrayList<>(elements);
            this.size = elements.size();
        }

        @Override
        public boolean hasNext() {
            return size > 0;
        }

        @Override
        public E next() {
            int idx = rand.nextInt(size);
            E e1 = elements.get(idx);
            E e2 = elements.get(size - 1);
            elements.set(idx, e2);
            size--;
            return e1;
        }
    }
}