net.myrrix.client.ClientRecommender.java Source code

Java tutorial

Introduction

Here is the source code for net.myrrix.client.ClientRecommender.java

Source

/*
 * Copyright Myrrix Ltd
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package net.myrrix.client;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Reader;
import java.io.Writer;
import java.net.Authenticator;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.PasswordAuthentication;
import java.net.URL;
import java.net.URLConnection;
import java.security.KeyManagementException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.concurrent.TimeUnit;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSession;
import javax.net.ssl.SSLSocketFactory;
import javax.net.ssl.TrustManagerFactory;

import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.base.Stopwatch;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.net.HostAndPort;
import com.google.common.net.HttpHeaders;
import com.google.common.net.MediaType;
import org.apache.commons.math3.util.FastMath;
import org.apache.commons.math3.util.Pair;
import org.apache.mahout.cf.taste.common.NoSuchItemException;
import org.apache.mahout.cf.taste.common.NoSuchUserException;
import org.apache.mahout.cf.taste.common.Refreshable;
import org.apache.mahout.cf.taste.common.TasteException;
import org.apache.mahout.cf.taste.impl.recommender.GenericRecommendedItem;
import org.apache.mahout.cf.taste.model.DataModel;
import org.apache.mahout.cf.taste.recommender.IDRescorer;
import org.apache.mahout.cf.taste.recommender.RecommendedItem;
import org.apache.mahout.cf.taste.recommender.Rescorer;
import org.apache.mahout.common.LongPair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import net.myrrix.common.collection.FastIDSet;
import net.myrrix.common.io.IOUtils;
import net.myrrix.common.LangUtils;
import net.myrrix.common.MyrrixRecommender;
import net.myrrix.common.NotReadyException;
import net.myrrix.common.random.RandomUtils;

/**
 * <p>An implementation of {@link MyrrixRecommender} which accesses a remote Serving Layer instance
 * over HTTP or HTTPS. This is like a local "handle" on the remote recommender.</p>
 *
 * <p>It is useful to note here, again, that the API methods {@link #setPreference(long, long)}
 * and {@link #removePreference(long, long)}, retained from Apache Mahout, have a somewhat different meaning
 * than in Mahout. They add to an association strength, rather than replace it. See the javadoc.</p>
 *
 * <p>There are a few advanced, system-wide parameters that can be set to affect how the client works.
 * These should not normally be used:</p>
 *
 * <ul>
 *   <li>{@code client.connection.close}: Causes the client to request no HTTP keep-alive. This can avoid
 *    running out of local sockets on the client side during load testing, but should not otherwise be set.</li>
 *   <li>{@code client.https.ignoreHost}: When using HTTPS, ignore the host specified in the certificate. This
 *    can make development easier, but must not be used in production.</li>
 * </ul>
 *
 * @author Sean Owen
 * @since 1.0
 */
public final class ClientRecommender implements MyrrixRecommender {

    private static final Logger log = LoggerFactory.getLogger(ClientRecommender.class);

    private static final Splitter COMMA = Splitter.on(',');
    private static final String IGNORE_HOSTNAME_KEY = "client.https.ignoreHost";
    private static final String CONNECTION_CLOSE_KEY = "client.connection.close";

    private static final Map<String, String> INGEST_REQUEST_PROPS;
    static {
        INGEST_REQUEST_PROPS = Maps.newHashMapWithExpectedSize(2);
        INGEST_REQUEST_PROPS.put(HttpHeaders.CONTENT_TYPE, MediaType.PLAIN_TEXT_UTF_8.toString());
        INGEST_REQUEST_PROPS.put(HttpHeaders.CONTENT_ENCODING, "gzip");
    }
    private static final String DESIRED_RESPONSE_CONTENT_TYPE = MediaType.CSV_UTF_8.withoutParameters().toString();

    private final MyrrixClientConfiguration config;
    private final boolean needAuthentication;
    private final boolean closeConnection;
    private final boolean ignoreHTTPSHost;
    private final List<List<HostAndPort>> partitions;

    /**
     * Instantiates a new recommender client with the given configuration
     *
     * @param config configuration to use with this client
     * @throws IOException if the HTTP client encounters an error during configuration
     */
    public ClientRecommender(MyrrixClientConfiguration config) throws IOException {
        Preconditions.checkNotNull(config);
        this.config = config;

        final String userName = config.getUserName();
        final String password = config.getPassword();
        needAuthentication = userName != null && password != null;
        if (needAuthentication) {
            Authenticator.setDefault(new Authenticator() {
                @Override
                protected PasswordAuthentication getPasswordAuthentication() {
                    return new PasswordAuthentication(userName, password.toCharArray());
                }
            });
        }

        if (config.getKeystoreFile() != null) {
            log.warn("A keystore file has been specified. "
                    + "This should only be done to accept self-signed certificates in development.");
            HttpsURLConnection.setDefaultSSLSocketFactory(buildSSLSocketFactory());
        }

        closeConnection = Boolean.valueOf(System.getProperty(CONNECTION_CLOSE_KEY));
        ignoreHTTPSHost = Boolean.valueOf(System.getProperty(IGNORE_HOSTNAME_KEY));

        partitions = config.getPartitions();
    }

    private SSLSocketFactory buildSSLSocketFactory() throws IOException {

        final HostnameVerifier defaultVerifier = HttpsURLConnection.getDefaultHostnameVerifier();
        HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier() {
            @Override
            public boolean verify(String hostname, SSLSession sslSession) {
                return ignoreHTTPSHost || "localhost".equals(hostname) || "127.0.0.1".equals(hostname)
                        || defaultVerifier.verify(hostname, sslSession);
            }
        });

        try {

            KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
            File trustStoreFile = config.getKeystoreFile().getAbsoluteFile();
            String password = config.getKeystorePassword();
            Preconditions.checkNotNull(password);

            InputStream in = new FileInputStream(trustStoreFile);
            try {
                keyStore.load(in, password.toCharArray());
            } finally {
                in.close();
            }

            TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
            tmf.init(keyStore);

            SSLContext ctx;
            try {
                ctx = SSLContext.getInstance("TLSv1.1"); // Java 7 only
            } catch (NoSuchAlgorithmException ignored) {
                log.info("TLSv1.1 unavailable, falling back to TLSv1");
                ctx = SSLContext.getInstance("TLSv1"); // Java 6       
                // This also seems to be necessary:
                if (System.getProperty("https.protocols") == null) {
                    System.setProperty("https.protocols", "TLSv1");
                }
            }
            ctx.init(null, tmf.getTrustManagers(), null);
            return ctx.getSocketFactory();

        } catch (NoSuchAlgorithmException nsae) {
            // can't happen?
            throw new IllegalStateException(nsae);
        } catch (KeyStoreException kse) {
            throw new IOException(kse);
        } catch (KeyManagementException kme) {
            throw new IOException(kme);
        } catch (CertificateException ce) {
            throw new IOException(ce);
        }
    }

    /**
     * @param replica host and port of replica to connect to
     * @param path URL to access (relative to context root)
     * @param method HTTP method to use
     */
    private HttpURLConnection buildConnectionToReplica(HostAndPort replica, String path, String method)
            throws IOException {
        return buildConnectionToReplica(replica, path, method, false, false, null);
    }

    /**
     * @param replica host and port of replica to connect to
     * @param path URL to access (relative to context root)
     * @param method HTTP method to use
     * @param doOutput if true, will need to write data into the request body
     * @param chunkedStreaming if true, use chunked streaming to accommodate a large upload, if possible
     * @param requestProperties additional request key/value pairs or {@code null} for none
     */
    private HttpURLConnection buildConnectionToReplica(HostAndPort replica, String path, String method,
            boolean doOutput, boolean chunkedStreaming, Map<String, String> requestProperties) throws IOException {
        String contextPath = config.getContextPath();
        if (contextPath != null) {
            path = '/' + contextPath + path;
        }
        String protocol = config.isSecure() ? "https" : "http";
        URL url;
        try {
            url = new URL(protocol, replica.getHostText(), replica.getPort(), path);
        } catch (MalformedURLException mue) {
            // can't happen
            throw new IllegalStateException(mue);
        }
        log.debug("{} {}", method, url);

        HttpURLConnection connection = (HttpURLConnection) url.openConnection();
        connection.setRequestMethod(method);
        connection.setDoInput(true);
        connection.setDoOutput(doOutput);
        connection.setUseCaches(false);
        connection.setAllowUserInteraction(false);
        connection.setRequestProperty(HttpHeaders.ACCEPT, DESIRED_RESPONSE_CONTENT_TYPE);
        if (closeConnection) {
            connection.setRequestProperty(HttpHeaders.CONNECTION, "close");
        }
        if (chunkedStreaming) {
            if (needAuthentication) {
                // Must buffer in memory if using authentication since it won't handle the authorization challenge
                log.debug("Authentication is enabled, so ingest data must be buffered in memory");
            } else {
                connection.setChunkedStreamingMode(0); // Use default buffer size
            }
        }
        if (requestProperties != null) {
            for (Map.Entry<String, String> entry : requestProperties.entrySet()) {
                connection.setRequestProperty(entry.getKey(), entry.getValue());
            }
        }
        return connection;
    }

    /**
     * @param unnormalizedID ID value that determines partition
     */
    private Iterable<HostAndPort> choosePartitionAndReplicas(long unnormalizedID) {
        List<HostAndPort> replicas = partitions.get(LangUtils.mod(unnormalizedID, partitions.size()));
        int numReplicas = replicas.size();
        if (numReplicas <= 1) {
            return replicas;
        }
        // Fix first replica; cycle through remainder in order since the remainder doesn't matter
        int currentReplica = LangUtils.mod(RandomUtils.md5HashToLong(unnormalizedID), numReplicas);
        Collection<HostAndPort> rotatedReplicas = Lists.newArrayListWithCapacity(numReplicas);
        for (int i = 0; i < numReplicas; i++) {
            rotatedReplicas.add(replicas.get(currentReplica));
            if (++currentReplica == numReplicas) {
                currentReplica = 0;
            }
        }
        return rotatedReplicas;
    }

    /**
     * Calls {@link #setPreference(long, long, float)} with value 1.0.
     */
    @Override
    public void setPreference(long userID, long itemID) throws TasteException {
        setPreference(userID, itemID, 1.0f);
    }

    @Override
    public void setPreference(long userID, long itemID, float value) throws TasteException {
        doSetOrRemovePreference(userID, itemID, value, true);
    }

    @Override
    public void removePreference(long userID, long itemID) throws TasteException {
        doSetOrRemovePreference(userID, itemID, 1.0f, false); // 1.0 is a dummy value that gets ignored
    }

    private void doSetOrRemovePreference(long userID, long itemID, float value, boolean set) throws TasteException {
        doSetOrRemove("/pref/" + userID + '/' + itemID, userID, value, set);
    }

    private void doSetOrRemove(String path, long unnormalizedID, float value, boolean set) throws TasteException {
        boolean sendValue = value != 1.0f;
        Map<String, String> requestProperties;
        byte[] bytes;
        if (sendValue) {
            requestProperties = Maps.newHashMapWithExpectedSize(2);
            bytes = Float.toString(value).getBytes(Charsets.UTF_8);
            requestProperties.put(HttpHeaders.CONTENT_TYPE, MediaType.PLAIN_TEXT_UTF_8.toString());
            requestProperties.put(HttpHeaders.CONTENT_LENGTH, Integer.toString(bytes.length));
        } else {
            requestProperties = null;
            bytes = null;
        }

        TasteException savedException = null;
        for (HostAndPort replica : choosePartitionAndReplicas(unnormalizedID)) {
            HttpURLConnection connection = null;
            try {
                connection = buildConnectionToReplica(replica, path, set ? "POST" : "DELETE", sendValue, false,
                        requestProperties);
                if (sendValue) {
                    OutputStream out = connection.getOutputStream();
                    out.write(bytes);
                    out.close();
                }
                // Should not be able to return Not Available status
                if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
                    throw new TasteException(connection.getResponseCode() + " " + connection.getResponseMessage());
                }
                return;
            } catch (TasteException te) {
                log.info("Can't access {} at {}: ({})", path, replica, te.toString());
                savedException = te;
            } catch (IOException ioe) {
                log.info("Can't access {} at {}: ({})", path, replica, ioe.toString());
                savedException = new TasteException(ioe);
            } finally {
                if (connection != null) {
                    connection.disconnect();
                }
            }
        }
        throw savedException;
    }

    @Override
    public void setUserTag(long userID, String tag) throws TasteException {
        setUserTag(userID, tag, 1.0f);
    }

    @Override
    public void setUserTag(long userID, String tag, float value) throws TasteException {
        Preconditions.checkNotNull(tag);
        Preconditions.checkArgument(!tag.isEmpty());
        doSetOrRemove("/tag/user/" + userID + '/' + IOUtils.urlEncode(tag), userID, value, true);
    }

    @Override
    public void setItemTag(String tag, long itemID) throws TasteException {
        setItemTag(tag, itemID, 1.0f);
    }

    @Override
    public void setItemTag(String tag, long itemID, float value) throws TasteException {
        setItemTag(tag, itemID, value, null);
    }

    /**
     * Like {@link #setItemTag(String, long, float)}, but allows caller to specify the user for
     * which the request is being made. This information does not directly affect the computation,
     * but affects <em>routing</em> of the request in a distributed context. This is always recommended
     * when there is a user in whose context the request is being made, as it will ensure that the
     * request can take into account all the latest information from the user, including a very new
     * item like {@code itemID}.
     */
    public void setItemTag(String tag, long itemID, float value, Long contextUserID) throws TasteException {
        Preconditions.checkNotNull(tag);
        Preconditions.checkArgument(!tag.isEmpty());
        long idToPartitionOn = contextUserID == null ? itemID : contextUserID;
        doSetOrRemove("/tag/item/" + itemID + '/' + IOUtils.urlEncode(tag), idToPartitionOn, value, true);
    }

    /**
     * @param userID user ID whose preference is to be estimated
     * @param itemID item ID to estimate preference for
     * @return an estimate of the strength of the association between the user and item. These values are the
     *  same as will be returned from {@link #recommend(long, int)}. They are opaque values and have no interpretation
     *  other than that larger means stronger. The values are typically in the range [0,1] but are not guaranteed
     *  to be so. Note that 0 will be returned if the user or item is not known in the data.
     * @throws NotReadyException if the recommender has no model available yet
     * @throws TasteException if another error occurs
     */
    @Override
    public float estimatePreference(long userID, long itemID) throws TasteException {
        return estimatePreferences(userID, itemID)[0];
    }

    @Override
    public float[] estimatePreferences(long userID, long... itemIDs) throws TasteException {
        StringBuilder urlPath = new StringBuilder();
        urlPath.append("/estimate/");
        urlPath.append(userID);
        for (long itemID : itemIDs) {
            urlPath.append('/').append(itemID);
        }

        TasteException savedException = null;
        for (HostAndPort replica : choosePartitionAndReplicas(userID)) {
            HttpURLConnection connection = null;
            try {
                connection = buildConnectionToReplica(replica, urlPath.toString(), "GET");
                switch (connection.getResponseCode()) {
                case HttpURLConnection.HTTP_OK:
                    BufferedReader reader = IOUtils.bufferStream(connection.getInputStream());
                    try {
                        float[] result = new float[itemIDs.length];
                        for (int i = 0; i < itemIDs.length; i++) {
                            result[i] = LangUtils.parseFloat(reader.readLine());
                        }
                        return result;
                    } finally {
                        reader.close();
                    }
                case HttpURLConnection.HTTP_UNAVAILABLE:
                    throw new NotReadyException();
                default:
                    throw new TasteException(connection.getResponseCode() + " " + connection.getResponseMessage());
                }
            } catch (TasteException te) {
                log.info("Can't access {} at {}: ({})", urlPath, replica, te.toString());
                savedException = te;
            } catch (IOException ioe) {
                log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString());
                savedException = new TasteException(ioe);
            } finally {
                if (connection != null) {
                    connection.disconnect();
                }
            }
        }
        throw savedException;
    }

    @Override
    public float estimateForAnonymous(long toItemID, long[] itemIDs) throws TasteException {
        return estimateForAnonymous(toItemID, itemIDs, null);
    }

    @Override
    public float estimateForAnonymous(long toItemID, long[] itemIDs, float[] values) throws TasteException {
        return estimateForAnonymous(toItemID, itemIDs, values, null);
    }

    public float estimateForAnonymous(long toItemID, long[] itemIDs, float[] values, Long contextUserID)
            throws TasteException {
        Preconditions.checkArgument(values == null || values.length == itemIDs.length,
                "Number of values doesn't match number of items");
        StringBuilder urlPath = new StringBuilder(32);
        urlPath.append("/estimateForAnonymous/");
        urlPath.append(toItemID);
        for (int i = 0; i < itemIDs.length; i++) {
            urlPath.append('/').append(itemIDs[i]);
            if (values != null) {
                urlPath.append('=').append(values[i]);
            }
        }

        // Requests are typically partitioned by user, but this request does not directly depend on a user.
        // If a user ID is supplied anyway, use it for partitioning since it will follow routing for other
        // requests related to that user. Otherwise just partition on the "to" item ID, which is at least
        // deterministic.
        long idToPartitionOn = contextUserID == null ? toItemID : contextUserID;

        TasteException savedException = null;
        for (HostAndPort replica : choosePartitionAndReplicas(idToPartitionOn)) {
            HttpURLConnection connection = null;
            try {
                connection = buildConnectionToReplica(replica, urlPath.toString(), "GET");
                switch (connection.getResponseCode()) {
                case HttpURLConnection.HTTP_OK:
                    BufferedReader reader = IOUtils.bufferStream(connection.getInputStream());
                    try {
                        return LangUtils.parseFloat(reader.readLine());
                    } finally {
                        reader.close();
                    }
                case HttpURLConnection.HTTP_NOT_FOUND:
                    throw new NoSuchItemException(Arrays.toString(itemIDs) + ' ' + toItemID);
                case HttpURLConnection.HTTP_UNAVAILABLE:
                    throw new NotReadyException();
                default:
                    throw new TasteException(connection.getResponseCode() + " " + connection.getResponseMessage());
                }
            } catch (TasteException te) {
                log.info("Can't access {} at {}: ({})", urlPath, replica, te.toString());
                savedException = te;
            } catch (IOException ioe) {
                log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString());
                savedException = new TasteException(ioe);
            } finally {
                if (connection != null) {
                    connection.disconnect();
                }
            }
        }
        throw savedException;
    }

    /**
     * Like {@link #recommend(long, int, boolean, IDRescorer)}, and sets {@code considerKnownItems} to {@code false}
     * and {@code rescorer} to {@code null}.
     */
    @Override
    public List<RecommendedItem> recommend(long userID, int howMany) throws TasteException {
        return recommend(userID, howMany, false, (String[]) null);
    }

    /**
     * @param userID user for which recommendations are to be computed
     * @param howMany desired number of recommendations
     * @param considerKnownItems if true, items that the user is already associated to are candidates
     *  for recommendation. Normally this is {@code false}.
     * @param rescorerParams optional parameters to send to the server's {@code RescorerProvider}
     * @return {@link List} of recommended {@link RecommendedItem}s, ordered from most strongly recommend to least
     * @throws NoSuchUserException if the user is not known in the model
     * @throws NotReadyException if the recommender has no model available yet
     * @throws TasteException if another error occurs
     * @throws UnsupportedOperationException if rescorer is not null
     */
    public List<RecommendedItem> recommend(long userID, int howMany, boolean considerKnownItems,
            String[] rescorerParams) throws TasteException {

        StringBuilder urlPath = new StringBuilder();
        urlPath.append("/recommend/");
        urlPath.append(userID);
        appendCommonQueryParams(howMany, considerKnownItems, rescorerParams, urlPath);

        TasteException savedException = null;
        for (HostAndPort replica : choosePartitionAndReplicas(userID)) {
            HttpURLConnection connection = null;
            try {
                connection = buildConnectionToReplica(replica, urlPath.toString(), "GET");
                switch (connection.getResponseCode()) {
                case HttpURLConnection.HTTP_OK:
                    return consumeItems(connection);
                case HttpURLConnection.HTTP_NOT_FOUND:
                    throw new NoSuchUserException(userID);
                case HttpURLConnection.HTTP_UNAVAILABLE:
                    throw new NotReadyException();
                default:
                    throw new TasteException(connection.getResponseCode() + " " + connection.getResponseMessage());
                }
            } catch (TasteException te) {
                log.info("Can't access {} at {}: ({})", urlPath, replica, te.toString());
                savedException = te;
            } catch (IOException ioe) {
                log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString());
                savedException = new TasteException(ioe);
            } finally {
                if (connection != null) {
                    connection.disconnect();
                }
            }
        }
        throw savedException;
    }

    private static List<RecommendedItem> consumeItems(URLConnection connection) throws IOException {
        List<RecommendedItem> result = Lists.newArrayList();
        BufferedReader reader = IOUtils.bufferStream(connection.getInputStream());
        try {
            CharSequence line;
            while ((line = reader.readLine()) != null) {
                Iterator<String> tokens = COMMA.split(line).iterator();
                long itemID = Long.parseLong(tokens.next());
                float value = LangUtils.parseFloat(tokens.next());
                result.add(new GenericRecommendedItem(itemID, value));
            }
        } finally {
            reader.close();
        }
        return result;
    }

    /**
     * @param userIDs users for which recommendations are to be computed
     * @param howMany desired number of recommendations
     * @param considerKnownItems if true, items that the user is already associated to are candidates
     *  for recommendation. Normally this is {@code false}.
     * @param rescorerParams optional parameters to send to the server's {@code RescorerProvider}
     * @return {@link List} of recommended {@link RecommendedItem}s, ordered from most strongly recommend to least
     * @throws NoSuchUserException if <em>none</em> of {@code userIDs} are known in the model. Otherwise unknown
     *  user IDs are ignored.
     * @throws NotReadyException if the recommender has no model available yet
     * @throws TasteException if another error occurs
     * @throws UnsupportedOperationException if rescorer is not null
     */
    public List<RecommendedItem> recommendToMany(long[] userIDs, int howMany, boolean considerKnownItems,
            String[] rescorerParams) throws TasteException {

        StringBuilder urlPath = new StringBuilder(32);
        urlPath.append("/recommendToMany");
        for (long userID : userIDs) {
            urlPath.append('/').append(userID);
        }
        appendCommonQueryParams(howMany, considerKnownItems, rescorerParams, urlPath);

        // Note that this assumes that all user IDs are on the same partition. It will fail at request
        // time if not since the partition of the first user doesn't contain the others.
        TasteException savedException = null;
        for (HostAndPort replica : choosePartitionAndReplicas(userIDs[0])) {
            HttpURLConnection connection = null;
            try {
                connection = buildConnectionToReplica(replica, urlPath.toString(), "GET");
                switch (connection.getResponseCode()) {
                case HttpURLConnection.HTTP_OK:
                    return consumeItems(connection);
                case HttpURLConnection.HTTP_NOT_FOUND:
                    throw new NoSuchUserException(Arrays.toString(userIDs));
                case HttpURLConnection.HTTP_UNAVAILABLE:
                    throw new NotReadyException();
                default:
                    throw new TasteException(connection.getResponseCode() + " " + connection.getResponseMessage());
                }
            } catch (TasteException te) {
                log.info("Can't access {} at {}: ({})", urlPath, replica, te.toString());
                savedException = te;
            } catch (IOException ioe) {
                log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString());
                savedException = new TasteException(ioe);
            } finally {
                if (connection != null) {
                    connection.disconnect();
                }
            }
        }
        throw savedException;
    }

    @Override
    public List<RecommendedItem> recommendToAnonymous(long[] itemIDs, int howMany) throws TasteException {
        return recommendToAnonymous(itemIDs, null, howMany);
    }

    @Override
    public List<RecommendedItem> recommendToAnonymous(long[] itemIDs, float[] values, int howMany)
            throws TasteException {
        return recommendToAnonymous(itemIDs, values, howMany, null, null);
    }

    /**
     * Like {@link #recommendToAnonymous(long[], float[], int)}, but allows caller to specify the user for
     * which the request is being made. This information does not directly affect the computation,
     * but affects <em>routing</em> of the request in a distributed context. This is always recommended
     * when there is a user in whose context the request is being made, as it will ensure that the
     * request can take into account all the latest information from the user, including very new
     * items that may be in {@code itemIDs}.
     */
    public List<RecommendedItem> recommendToAnonymous(long[] itemIDs, float[] values, int howMany,
            String[] rescorerParams, Long contextUserID) throws TasteException {
        return anonymousOrSimilar(itemIDs, values, howMany, "/recommendToAnonymous", rescorerParams, contextUserID);
    }

    @Override
    public List<RecommendedItem> mostPopularItems(int howMany) throws TasteException {
        StringBuilder urlPath = new StringBuilder(32);
        urlPath.append("/mostPopularItems");
        appendCommonQueryParams(howMany, false, null, urlPath);

        // Always send to partition 0 for consistency    
        TasteException savedException = null;
        for (HostAndPort replica : choosePartitionAndReplicas(0L)) {
            HttpURLConnection connection = null;
            try {
                connection = buildConnectionToReplica(replica, urlPath.toString(), "GET");
                switch (connection.getResponseCode()) {
                case HttpURLConnection.HTTP_OK:
                    return consumeItems(connection);
                case HttpURLConnection.HTTP_UNAVAILABLE:
                    throw new NotReadyException();
                default:
                    throw new TasteException(connection.getResponseCode() + " " + connection.getResponseMessage());
                }
            } catch (TasteException te) {
                log.info("Can't access {} at {}: ({})", urlPath, replica, te.toString());
                savedException = te;
            } catch (IOException ioe) {
                log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString());
                savedException = new TasteException(ioe);
            } finally {
                if (connection != null) {
                    connection.disconnect();
                }
            }
        }
        throw savedException;
    }

    /**
     * Computes items most similar to an item or items. The returned items have the highest average similarity
     * to the given items.
     *
     * @param itemIDs items for which most similar items are required
     * @param howMany maximum number of similar items to return; fewer may be returned
     * @return {@link RecommendedItem}s representing the top recommendations for the user, ordered by quality,
     *  descending. The score associated to it is an opaque value. Larger means more similar, but no further
     *  interpretation may necessarily be applied.
     * @throws NoSuchItemException if <em>none</em> of {@code itemIDs} exist in the model. Otherwise, unknown
     *  items are ignored.
     * @throws NotReadyException if the recommender has no model available yet
     * @throws TasteException if another error occurs
     */
    @Override
    public List<RecommendedItem> mostSimilarItems(long[] itemIDs, int howMany) throws TasteException {
        return mostSimilarItems(itemIDs, howMany, null, null);
    }

    /**
     * Like {@link #mostSimilarItems(long[], int)}, but allows caller to specify the user for which the request
     * is being made. This information does not directly affect the computation, but affects <em>routing</em>
     * of the request in a distributed context. This is always recommended when there is a user in whose context
     * the request is being made, as it will ensure that the request can take into account all the latest information
     * from the user, including very new items that may be in {@code itemIDs}.
     */
    public List<RecommendedItem> mostSimilarItems(long[] itemIDs, int howMany, String[] rescorerParams,
            Long contextUserID) throws TasteException {
        return anonymousOrSimilar(itemIDs, null, howMany, "/similarity", rescorerParams, contextUserID);
    }

    private List<RecommendedItem> anonymousOrSimilar(long[] itemIDs, float[] values, int howMany, String path,
            String[] rescorerParams, Long contextUserID) throws TasteException {
        Preconditions.checkArgument(values == null || values.length == itemIDs.length,
                "Number of values doesn't match number of items");
        StringBuilder urlPath = new StringBuilder();
        urlPath.append(path);
        for (int i = 0; i < itemIDs.length; i++) {
            urlPath.append('/').append(itemIDs[i]);
            if (values != null) {
                urlPath.append('=').append(values[i]);
            }
        }
        appendCommonQueryParams(howMany, false, rescorerParams, urlPath);

        // Requests are typically partitioned by user, but this request does not directly depend on a user.
        // If a user ID is supplied anyway, use it for partitioning since it will follow routing for other
        // requests related to that user. Otherwise just partition on (first0 item ID, which is at least
        // deterministic.
        long idToPartitionOn = contextUserID == null ? itemIDs[0] : contextUserID;

        TasteException savedException = null;
        for (HostAndPort replica : choosePartitionAndReplicas(idToPartitionOn)) {
            HttpURLConnection connection = null;
            try {
                connection = buildConnectionToReplica(replica, urlPath.toString(), "GET");
                switch (connection.getResponseCode()) {
                case HttpURLConnection.HTTP_OK:
                    return consumeItems(connection);
                case HttpURLConnection.HTTP_NOT_FOUND:
                    throw new NoSuchItemException(Arrays.toString(itemIDs));
                case HttpURLConnection.HTTP_UNAVAILABLE:
                    throw new NotReadyException();
                default:
                    throw new TasteException(connection.getResponseCode() + " " + connection.getResponseMessage());
                }
            } catch (TasteException te) {
                log.info("Can't access {} at {}: ({})", urlPath, replica, te.toString());
                savedException = te;
            } catch (IOException ioe) {
                log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString());
                savedException = new TasteException(ioe);
            } finally {
                if (connection != null) {
                    connection.disconnect();
                }
            }
        }
        throw savedException;
    }

    /**
     * One-argument version of {@link #mostSimilarItems(long[], int)}.
     */
    @Override
    public List<RecommendedItem> mostSimilarItems(long itemID, int howMany) throws TasteException {
        return mostSimilarItems(new long[] { itemID }, howMany);
    }

    @Override
    public float[] similarityToItem(long toItemID, long... itemIDs) throws TasteException {
        return similarityToItem(toItemID, itemIDs, null);
    }

    /**
     * Like {@link #similarityToItem(long, long[])}, but allows caller to specify the user for which the request
     * is being made. This information does not directly affect the computation, but affects <em>routing</em>
     * of the request in a distributed context. This is always recommended when there is a user in whose context
     * the request is being made, as it will ensure that the request can take into account all the latest information
     * from the user, including very new items that may be in {@code itemIDs}.
     */
    public float[] similarityToItem(long toItemID, long[] itemIDs, Long contextUserID) throws TasteException {
        StringBuilder urlPath = new StringBuilder(32);
        urlPath.append("/similarityToItem/");
        urlPath.append(toItemID);
        for (long itemID : itemIDs) {
            urlPath.append('/').append(itemID);
        }

        // Requests are typically partitioned by user, but this request does not directly depend on a user.
        // If a user ID is supplied anyway, use it for partitioning since it will follow routing for other
        // requests related to that user. Otherwise just partition on (first0 item ID, which is at least
        // deterministic.
        long idToPartitionOn = contextUserID == null ? itemIDs[0] : contextUserID;

        TasteException savedException = null;
        for (HostAndPort replica : choosePartitionAndReplicas(idToPartitionOn)) {
            HttpURLConnection connection = null;
            try {
                connection = buildConnectionToReplica(replica, urlPath.toString(), "GET");
                switch (connection.getResponseCode()) {
                case HttpURLConnection.HTTP_OK:
                    BufferedReader reader = IOUtils.bufferStream(connection.getInputStream());
                    try {
                        float[] result = new float[itemIDs.length];
                        for (int i = 0; i < itemIDs.length; i++) {
                            result[i] = LangUtils.parseFloat(reader.readLine());
                        }
                        return result;
                    } finally {
                        reader.close();
                    }
                case HttpURLConnection.HTTP_NOT_FOUND:
                    throw new NoSuchItemException(connection.getResponseMessage());
                case HttpURLConnection.HTTP_UNAVAILABLE:
                    throw new NotReadyException();
                default:
                    throw new TasteException(connection.getResponseCode() + " " + connection.getResponseMessage());
                }
            } catch (TasteException te) {
                log.info("Can't access {} at {}: ({})", urlPath, replica, te.toString());
                savedException = te;
            } catch (IOException ioe) {
                log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString());
                savedException = new TasteException(ioe);
            } finally {
                if (connection != null) {
                    connection.disconnect();
                }
            }
        }
        throw savedException;
    }

    /**
     * <p>Lists the items that were most influential in recommending a given item to a given user. Exactly how this
     * is determined is left to the implementation, but, generally this will return items that the user prefers
     * and that are similar to the given item.</p>
     *
     * <p>These values by which the results are ordered are opaque values and have no interpretation
     * other than that larger means stronger.</p>
     *
     * @param userID ID of user who was recommended the item
     * @param itemID ID of item that was recommended
     * @param howMany maximum number of items to return
     * @return {@link List} of {@link RecommendedItem}, ordered from most influential in recommended the given
     *  item to least
     * @throws NoSuchUserException if the user is not known in the model
     * @throws NoSuchItemException if the item is not known in the model
     * @throws NotReadyException if the recommender has no model available yet
     * @throws TasteException if another error occurs
     */
    @Override
    public List<RecommendedItem> recommendedBecause(long userID, long itemID, int howMany) throws TasteException {
        String urlPath = "/because/" + userID + '/' + itemID + "?howMany=" + howMany;

        TasteException savedException = null;
        for (HostAndPort replica : choosePartitionAndReplicas(userID)) {
            HttpURLConnection connection = null;
            try {
                connection = buildConnectionToReplica(replica, urlPath, "GET");
                switch (connection.getResponseCode()) {
                case HttpURLConnection.HTTP_OK:
                    return consumeItems(connection);
                case HttpURLConnection.HTTP_NOT_FOUND:
                    String connectionMessage = connection.getResponseMessage();
                    if (connectionMessage != null
                            && connectionMessage.contains(NoSuchUserException.class.getSimpleName())) {
                        throw new NoSuchUserException(userID);
                    } else {
                        throw new NoSuchItemException(itemID);
                    }
                case HttpURLConnection.HTTP_UNAVAILABLE:
                    throw new NotReadyException();
                default:
                    throw new TasteException(connection.getResponseCode() + " " + connection.getResponseMessage());
                }
            } catch (TasteException te) {
                log.info("Can't access {} at {}: ({})", urlPath, replica, te.toString());
                savedException = te;
            } catch (IOException ioe) {
                log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString());
                savedException = new TasteException(ioe);
            } finally {
                if (connection != null) {
                    connection.disconnect();
                }
            }
        }
        throw savedException;
    }

    @Override
    public void ingest(File file) throws TasteException {
        Reader reader = null;
        try {
            reader = IOUtils.openReaderMaybeDecompressing(file);
            ingest(reader);
        } catch (IOException ioe) {
            throw new TasteException(ioe);
        } finally {
            try {
                reader.close();
            } catch (IOException e) {
                // Can't happen, continue
            }
        }
    }

    @Override
    public void ingest(Reader reader) throws TasteException {
        Map<Integer, Pair<Writer, HttpURLConnection>> writersAndConnections = Maps
                .newHashMapWithExpectedSize(partitions.size());
        BufferedReader buffered = IOUtils.buffer(reader);
        try {
            try {
                String line;
                while ((line = buffered.readLine()) != null) {
                    if (line.isEmpty() || line.charAt(0) == '#') {
                        continue;
                    }
                    long userID;
                    try {
                        userID = Long.parseLong(COMMA.split(line).iterator().next());
                    } catch (NoSuchElementException nsee) {
                        throw new TasteException(nsee);
                    } catch (NumberFormatException nfe) {
                        throw new TasteException(nfe);
                    }
                    int partition = LangUtils.mod(userID, partitions.size());
                    Pair<Writer, HttpURLConnection> writerAndConnection = writersAndConnections.get(partition);
                    if (writerAndConnection == null) {
                        HttpURLConnection connection = buildConnectionToAReplica(partition);
                        Writer writer = IOUtils.buildGZIPWriter(connection.getOutputStream());
                        writerAndConnection = new Pair<Writer, HttpURLConnection>(writer, connection);
                        writersAndConnections.put(partition, writerAndConnection);
                    }
                    Writer writer = writerAndConnection.getFirst();
                    writer.write(line);
                    writer.write('\n');
                }
                for (Pair<Writer, HttpURLConnection> writerAndConnection : writersAndConnections.values()) {
                    // Want to know of output stream close failed -- maybe failed to write
                    writerAndConnection.getFirst().close();
                    HttpURLConnection connection = writerAndConnection.getSecond();
                    if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
                        throw new TasteException(
                                connection.getResponseCode() + " " + connection.getResponseMessage());
                    }
                }
            } finally {
                for (Pair<Writer, HttpURLConnection> writerAndConnection : writersAndConnections.values()) {
                    writerAndConnection.getFirst().close();
                    writerAndConnection.getSecond().disconnect();
                }
            }
        } catch (IOException ioe) {
            throw new TasteException(ioe);
        }
    }

    private HttpURLConnection buildConnectionToAReplica(int partition) throws TasteException {
        String urlPath = "/ingest";

        TasteException savedException = null;
        for (HostAndPort replica : partitions.get(partition)) {
            HttpURLConnection connection = null;
            try {
                connection = buildConnectionToReplica(replica, urlPath, "POST", true, true, INGEST_REQUEST_PROPS);
                connection.connect();
                return connection;
            } catch (IOException ioe) {
                log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString());
                savedException = new TasteException(ioe);
            } finally {
                if (connection != null) {
                    connection.disconnect();
                }
            }
        }
        throw savedException;
    }

    @Deprecated
    @Override
    public void refresh(Collection<Refreshable> alreadyRefreshed) {
        if (alreadyRefreshed != null) {
            log.warn("Ignoring argument {}", alreadyRefreshed);
        }
        refresh();
    }

    /**
     * Requests that the Serving Layer recompute its models. This is a request, and may or may not result
     * in an update.
     */
    @Override
    public void refresh() {
        int numPartitions = partitions.size();
        for (int i = 0; i < numPartitions; i++) {
            refreshPartition(i);
        }
    }

    private void refreshPartition(int partition) {
        String urlPath = "/refresh";

        for (HostAndPort replica : partitions.get(partition)) {
            HttpURLConnection connection = null;
            try {
                connection = buildConnectionToReplica(replica, urlPath, "POST");
                if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
                    log.warn("Unable to refresh partition {} ({} {}); continuing", partition,
                            connection.getResponseCode(), connection.getResponseMessage());
                    // Yes, continue
                }
            } catch (IOException ioe) {
                log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString());
            } finally {
                if (connection != null) {
                    connection.disconnect();
                }
            }
        }
    }

    /**
     * Not available. The client does not directly use any {@link DataModel}.
     *
     * @throws UnsupportedOperationException
     * @deprecated do not call
     */
    @Deprecated
    @Override
    public DataModel getDataModel() {
        throw new UnsupportedOperationException();
    }

    /**
     * {@link Rescorer}s are not available at this time in the model.
     *
     * @return {@link #recommend(long, int)} if rescorer is null
     * @throws UnsupportedOperationException otherwise
     * @deprecated use {@link #recommend(long, int)} instead
     */
    @Deprecated
    @Override
    public List<RecommendedItem> recommend(long userID, int howMany, IDRescorer rescorer) throws TasteException {
        if (rescorer != null) {
            throw new UnsupportedOperationException();
        }
        return recommend(userID, howMany);
    }

    /**
     * {@link Rescorer}s are not available at this time in the model.
     *
     * @return {@link #recommend(long, int, boolean, String[])} if rescorer is null
     * @throws UnsupportedOperationException otherwise
     * @deprecated use {@link #recommend(long, int, boolean, String[])} instead
     */
    @Deprecated
    @Override
    public List<RecommendedItem> recommend(long userID, int howMany, boolean considerKnownItems,
            IDRescorer rescorer) throws TasteException {
        if (rescorer != null) {
            throw new UnsupportedOperationException();
        }
        return recommend(userID, howMany, considerKnownItems, (String[]) null);
    }

    @Deprecated
    @Override
    public List<RecommendedItem> recommendToMany(long[] userIDs, int howMany, boolean considerKnownItems,
            IDRescorer rescorer) throws TasteException {
        if (rescorer != null) {
            throw new UnsupportedOperationException();
        }
        return recommendToMany(userIDs, howMany, considerKnownItems, (String[]) null);
    }

    /**
     * Note that {@link IDRescorer} is not supported in the client now and must be null.
     *
     * @return {@link #recommendToAnonymous(long[], int)} if rescorer is null
     * @throws UnsupportedOperationException otherwise
     * @deprecated use {@link #recommendToAnonymous(long[], int)} instead
     */
    @Deprecated
    @Override
    public List<RecommendedItem> recommendToAnonymous(long[] itemIDs, int howMany, IDRescorer rescorer)
            throws TasteException {
        if (rescorer != null) {
            throw new UnsupportedOperationException();
        }
        return recommendToAnonymous(itemIDs, howMany);
    }

    /**
     * Note that {@link IDRescorer} is not supported in the client now and must be null.
     *
     * @return {@link #recommendToAnonymous(long[], float[], int)} if rescorer is null
     * @throws UnsupportedOperationException otherwise
     * @deprecated use {@link #recommendToAnonymous(long[], float[], int)} instead
     */
    @Deprecated
    @Override
    public List<RecommendedItem> recommendToAnonymous(long[] itemIDs, float[] values, int howMany,
            IDRescorer rescorer) throws TasteException {
        if (rescorer != null) {
            throw new UnsupportedOperationException();
        }
        return recommendToAnonymous(itemIDs, values, howMany);
    }

    /**
     * Note that {@link IDRescorer} is not supported in the client now and must be null.
     *
     * @return {@link #mostPopularItems(int)} if rescorer is null
     * @throws UnsupportedOperationException otherwise
     * @deprecated use {@link #mostPopularItems(int)} instead
     */
    @Deprecated
    @Override
    public List<RecommendedItem> mostPopularItems(int howMany, IDRescorer rescorer) throws TasteException {
        if (rescorer != null) {
            throw new UnsupportedOperationException();
        }
        return mostPopularItems(howMany);
    }

    /**
     * {@link Rescorer}s are not available at this time in the model.
     *
     * @return {@link #mostSimilarItems(long, int)} if rescorer is null
     * @throws UnsupportedOperationException otherwise
     * @deprecated use {@link #mostSimilarItems(long, int)} instead
     */
    @Deprecated
    @Override
    public List<RecommendedItem> mostSimilarItems(long itemID, int howMany, Rescorer<LongPair> rescorer)
            throws TasteException {
        if (rescorer != null) {
            throw new UnsupportedOperationException();
        }
        return mostSimilarItems(itemID, howMany);
    }

    /**
     * {@link Rescorer}s are not available at this time in the model.
     *
     * @return {@link #mostSimilarItems(long[], int)} if rescorer is null
     * @throws UnsupportedOperationException otherwise
     * @deprecated use {@link #mostSimilarItems(long[], int)} instead
     */
    @Deprecated
    @Override
    public List<RecommendedItem> mostSimilarItems(long[] itemIDs, int howMany, Rescorer<LongPair> rescorer)
            throws TasteException {
        if (rescorer != null) {
            throw new UnsupportedOperationException();
        }
        return mostSimilarItems(itemIDs, howMany);
    }

    /**
     * {@code excludeItemIfNotSimilarToAll} is not applicable in this implementation.
     *
     * @return {@link #mostSimilarItems(long[], int)} if excludeItemIfNotSimilarToAll is false
     * @throws UnsupportedOperationException otherwise
     * @deprecated use {@link #mostSimilarItems(long[], int)} instead
     */
    @Deprecated
    @Override
    public List<RecommendedItem> mostSimilarItems(long[] itemIDs, int howMany, boolean excludeItemIfNotSimilarToAll)
            throws TasteException {
        if (excludeItemIfNotSimilarToAll) {
            throw new UnsupportedOperationException();
        }
        return mostSimilarItems(itemIDs, howMany);
    }

    /**
     * {@link Rescorer}s are not available at this time in the model.
     * {@code excludeItemIfNotSimilarToAll} is not applicable in this implementation.
     *
     * @return {@link #mostSimilarItems(long[], int)} if excludeItemIfNotSimilarToAll is false and rescorer is null
     * @throws UnsupportedOperationException otherwise
     * @deprecated use {@link #mostSimilarItems(long[], int)} instead
     */
    @Deprecated
    @Override
    public List<RecommendedItem> mostSimilarItems(long[] itemIDs, int howMany, Rescorer<LongPair> rescorer,
            boolean excludeItemIfNotSimilarToAll) throws TasteException {
        if (excludeItemIfNotSimilarToAll || rescorer != null) {
            throw new UnsupportedOperationException();
        }
        return mostSimilarItems(itemIDs, howMany);
    }

    @Override
    public boolean isReady() throws TasteException {
        int numPartitions = partitions.size();
        for (int i = 0; i < numPartitions; i++) {
            if (!isPartitionReady(i)) {
                return false;
            }
        }
        return true;
    }

    private boolean isPartitionReady(int partition) throws TasteException {
        String urlPath = "/ready";

        TasteException savedException = null;
        for (HostAndPort replica : partitions.get(partition)) {
            HttpURLConnection connection = null;
            try {
                connection = buildConnectionToReplica(replica, urlPath, "HEAD");
                switch (connection.getResponseCode()) {
                case HttpURLConnection.HTTP_OK:
                    return true;
                case HttpURLConnection.HTTP_UNAVAILABLE:
                    return false;
                default:
                    throw new TasteException(connection.getResponseCode() + " " + connection.getResponseMessage());
                }
            } catch (TasteException te) {
                log.info("Can't access {} at {}: ({})", urlPath, replica, te.toString());
                savedException = te;
            } catch (IOException ioe) {
                log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString());
                savedException = new TasteException(ioe);
            } finally {
                if (connection != null) {
                    connection.disconnect();
                }
            }
        }
        throw savedException;
    }

    @Override
    public void await() throws TasteException, InterruptedException {
        while (!isReady()) {
            Thread.sleep(1000L);
        }
    }

    @Override
    public boolean await(long time, TimeUnit unit) throws TasteException, InterruptedException {
        Preconditions.checkArgument(time >= 0L, "time must be positive: {}", time);
        Preconditions.checkNotNull(unit);
        long waitForMS = TimeUnit.MILLISECONDS.convert(time, unit);
        long waitIntervalMS = FastMath.min(1000L, waitForMS);
        Stopwatch stopwatch = new Stopwatch().start();
        while (!isReady()) {
            if (stopwatch.elapsed(TimeUnit.MILLISECONDS) > waitForMS) {
                return false;
            }
            Thread.sleep(waitIntervalMS);
        }
        return true;
    }

    @Override
    public FastIDSet getAllUserIDs() throws TasteException {
        FastIDSet result = new FastIDSet();
        int numPartitions = partitions.size();
        for (int i = 0; i < numPartitions; i++) {
            getAllIDsFromPartition(i, true, result);
        }
        return result;
    }

    @Override
    public FastIDSet getAllItemIDs() throws TasteException {
        // Yes, loop over all partitions. Most item IDs will be returned from all partitions but it's
        // possible for some to exist only on a few.
        FastIDSet result = new FastIDSet();
        int numPartitions = partitions.size();
        for (int i = 0; i < numPartitions; i++) {
            getAllIDsFromPartition(i, false, result);
        }
        return result;
    }

    private void getAllIDsFromPartition(int partition, boolean user, FastIDSet result) throws TasteException {
        String urlPath = '/' + (user ? "user" : "item") + "/allIDs";

        TasteException savedException = null;
        for (HostAndPort replica : partitions.get(partition)) {
            HttpURLConnection connection = null;
            try {
                connection = buildConnectionToReplica(replica, urlPath, "GET");
                switch (connection.getResponseCode()) {
                case HttpURLConnection.HTTP_OK:
                    consumeIDs(connection, result);
                    return;
                case HttpURLConnection.HTTP_UNAVAILABLE:
                    throw new NotReadyException();
                default:
                    throw new TasteException(connection.getResponseCode() + " " + connection.getResponseMessage());
                }
            } catch (TasteException te) {
                log.info("Can't access {} at {}: ({})", urlPath, replica, te.toString());
                savedException = te;
            } catch (IOException ioe) {
                log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString());
                savedException = new TasteException(ioe);
            } finally {
                if (connection != null) {
                    connection.disconnect();
                }
            }
        }
        throw savedException;
    }

    private static void consumeIDs(URLConnection connection, FastIDSet result) throws IOException {
        BufferedReader reader = IOUtils.bufferStream(connection.getInputStream());
        try {
            String line;
            while ((line = reader.readLine()) != null) {
                result.add(Long.parseLong(line));
            }
        } finally {
            reader.close();
        }
    }

    @Override
    public int getNumUserClusters() throws TasteException {
        return getNumClusters(true);
    }

    @Override
    public int getNumItemClusters() throws TasteException {
        return getNumClusters(false);
    }

    private int getNumClusters(boolean user) throws TasteException {
        String urlPath = '/' + (user ? "user" : "item") + "/clusters/count";

        TasteException savedException = null;
        for (HostAndPort replica : choosePartitionAndReplicas(0L)) {
            HttpURLConnection connection = null;
            try {
                connection = buildConnectionToReplica(replica, urlPath, "GET");
                switch (connection.getResponseCode()) {
                case HttpURLConnection.HTTP_OK:
                    BufferedReader reader = IOUtils.bufferStream(connection.getInputStream());
                    try {
                        return Integer.parseInt(reader.readLine());
                    } finally {
                        reader.close();
                    }
                case HttpURLConnection.HTTP_NOT_IMPLEMENTED:
                    throw new UnsupportedOperationException();
                case HttpURLConnection.HTTP_UNAVAILABLE:
                    throw new NotReadyException();
                default:
                    throw new TasteException(connection.getResponseCode() + " " + connection.getResponseMessage());
                }
            } catch (TasteException te) {
                log.info("Can't access {} at {}: ({})", urlPath, replica, te.toString());
                savedException = te;
            } catch (IOException ioe) {
                log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString());
                savedException = new TasteException(ioe);
            } finally {
                if (connection != null) {
                    connection.disconnect();
                }
            }
        }
        throw savedException;
    }

    @Override
    public FastIDSet getUserCluster(int n) throws TasteException {
        return getCluster(n, true);
    }

    @Override
    public FastIDSet getItemCluster(int n) throws TasteException {
        return getCluster(n, false);
    }

    private FastIDSet getCluster(int n, boolean user) throws TasteException {
        String urlPath = '/' + (user ? "user" : "item") + "/clusters/" + n;

        TasteException savedException = null;
        for (HostAndPort replica : choosePartitionAndReplicas(0L)) {
            HttpURLConnection connection = null;
            try {
                connection = buildConnectionToReplica(replica, urlPath, "GET");
                switch (connection.getResponseCode()) {
                case HttpURLConnection.HTTP_OK:
                    FastIDSet members = new FastIDSet();
                    consumeIDs(connection, members);
                    return members;
                case HttpURLConnection.HTTP_NOT_IMPLEMENTED:
                    throw new UnsupportedOperationException();
                case HttpURLConnection.HTTP_UNAVAILABLE:
                    throw new NotReadyException();
                default:
                    throw new TasteException(connection.getResponseCode() + " " + connection.getResponseMessage());
                }
            } catch (TasteException te) {
                log.info("Can't access {} at {}: ({})", urlPath, replica, te.toString());
                savedException = te;
            } catch (IOException ioe) {
                log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString());
                savedException = new TasteException(ioe);
            } finally {
                if (connection != null) {
                    connection.disconnect();
                }
            }
        }
        throw savedException;
    }

    private static void appendCommonQueryParams(int howMany, boolean considerKnownItems, String[] rescorerParams,
            StringBuilder urlPath) {
        urlPath.append("?howMany=").append(howMany);
        if (considerKnownItems) {
            urlPath.append("&considerKnownItems=true");
        }
        if (rescorerParams != null) {
            for (String rescorerParam : rescorerParams) {
                urlPath.append("&rescorerParams=").append(IOUtils.urlEncode(rescorerParam));
            }
        }
    }

}