Java tutorial
/** * Copyright 2013 Twitter, Inc. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. **/ package com.twitter.hbc.httpclient; import java.io.IOException; import java.net.UnknownHostException; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import javax.annotation.Nullable; import org.apache.http.StatusLine; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpUriRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.twitter.hbc.RateTracker; import com.twitter.hbc.ReconnectionManager; import com.twitter.hbc.core.Hosts; import com.twitter.hbc.core.HttpConstants; import com.twitter.hbc.core.StatsReporter; import com.twitter.hbc.core.endpoint.StreamingEndpoint; import com.twitter.hbc.core.event.ConnectionEvent; import com.twitter.hbc.core.event.Event; import com.twitter.hbc.core.event.EventType; import com.twitter.hbc.core.event.HttpResponseEvent; import com.twitter.hbc.core.processor.HosebirdMessageProcessor; import com.twitter.hbc.httpclient.auth.Authentication; /** * Thread-safe. * TODO: better name?!? */ class ClientBase implements Runnable { private final static Logger logger = LoggerFactory.getLogger(ClientBase.class); private final String name; private final HttpClient client; private final StreamingEndpoint endpoint; private final Hosts hosts; private final Authentication auth; private final HosebirdMessageProcessor processor; private final ReconnectionManager reconnectionManager; private final AtomicReference<Event> exitEvent; private final CountDownLatch isRunning; private final RateTracker rateTracker; private final BlockingQueue<Event> eventsQueue; private final StatsReporter statsReporter; private final AtomicBoolean connectionEstablished; private final AtomicBoolean reconnect; ClientBase(String name, HttpClient client, Hosts hosts, StreamingEndpoint endpoint, Authentication auth, HosebirdMessageProcessor processor, ReconnectionManager manager, RateTracker rateTracker) { this(name, client, hosts, endpoint, auth, processor, manager, rateTracker, null); } // TODO: support setting some http timeouts? ClientBase(String name, HttpClient client, Hosts hosts, StreamingEndpoint endpoint, Authentication auth, HosebirdMessageProcessor processor, ReconnectionManager manager, RateTracker rateTracker, @Nullable BlockingQueue<Event> eventsQueue) { this.client = Preconditions.checkNotNull(client); this.name = Preconditions.checkNotNull(name); this.endpoint = Preconditions.checkNotNull(endpoint); this.hosts = Preconditions.checkNotNull(hosts); this.auth = Preconditions.checkNotNull(auth); this.processor = Preconditions.checkNotNull(processor); this.reconnectionManager = Preconditions.checkNotNull(manager); this.rateTracker = Preconditions.checkNotNull(rateTracker); this.eventsQueue = eventsQueue; this.exitEvent = new AtomicReference<Event>(); this.isRunning = new CountDownLatch(1); this.statsReporter = new StatsReporter(); this.connectionEstablished = new AtomicBoolean(false); this.reconnect = new AtomicBoolean(false); } @Override public void run() { // establish the initial connection // if connection fails due to auth or some other 400, stop immediately // if connection fails due to a 500, back off and retry // if no response or other code, stop immediately // begin reading from the stream // while the stop signal hasn't been sent, and no IOException from processor, keep processing // if IOException, time to restart the connection: // handle http connection cleanup // do some backoff, set backfill // if stop signal set, time to kill/clean the connection, and end this thread. try { if (client instanceof RestartableHttpClient) { ((RestartableHttpClient) client).setup(); } rateTracker.start(); while (!isDone()) { String host = hosts.nextHost(); if (host == null) { setExitStatus(new Event(EventType.STOPPED_BY_ERROR, "No hosts available")); break; } double rate = rateTracker.getCurrentRateSeconds(); if (!Double.isNaN(rate)) { endpoint.setBackfillCount(reconnectionManager.estimateBackfill(rate)); } HttpUriRequest request = HttpConstants.constructRequest(host, endpoint, auth); if (request != null) { String postContent = null; if (endpoint.getHttpMethod().equalsIgnoreCase(HttpConstants.HTTP_POST)) { postContent = endpoint.getPostParamString(); } auth.signRequest(request, postContent); Connection conn = new Connection(client, processor); StatusLine status = establishConnection(conn, request); if (handleConnectionResult(status)) { rateTracker.resume(); processConnectionData(conn); rateTracker.pause(); } logger.info("{} Done processing, preparing to close connection", name); conn.close(); } else { addEvent(new Event(EventType.CONNECTION_ERROR, String.format("Error creating request: %s, %s, %s", endpoint.getHttpMethod(), host, endpoint.getURI()))); } } } catch (Throwable e) { logger.warn(name + " Uncaught exception", e); Exception laundered = (e instanceof Exception) ? (Exception) e : new RuntimeException(e); setExitStatus(new Event(EventType.STOPPED_BY_ERROR, laundered)); } finally { rateTracker.stop(); logger.info("{} Shutting down httpclient connection manager", name); client.getConnectionManager().shutdown(); isRunning.countDown(); } } @Nullable() @VisibleForTesting StatusLine establishConnection(Connection conn, HttpUriRequest request) { logger.info("{} Establishing a connection", name); // establish connection StatusLine status = null; try { addEvent(new ConnectionEvent(EventType.CONNECTION_ATTEMPT, request)); status = conn.connect(request); } catch (UnknownHostException e) { // banking on some httpHosts.nextHost() being legitimate, or else this connection will fail. logger.warn("{} Unknown host - {}", name, request.getURI().getHost()); addEvent(new Event(EventType.CONNECTION_ERROR, e)); } catch (IOException e) { logger.warn("{} IOException caught when establishing connection to {}", name, request.getURI()); addEvent(new Event(EventType.CONNECTION_ERROR, e)); reconnectionManager.handleLinearBackoff(); } catch (Exception e) { logger.error(String.format("%s Unknown exception while establishing connection to %s", name, request.getURI()), e); setExitStatus(new Event(EventType.STOPPED_BY_ERROR, e)); } return status; } /** * @return whether a successful connection has been established */ @VisibleForTesting boolean handleConnectionResult(@Nullable StatusLine statusLine) { statsReporter.incrNumConnects(); if (statusLine == null) { logger.warn("{} failed to establish connection properly", name); addEvent(new Event(EventType.CONNECTION_ERROR, "Failed to establish connection properly")); return false; } int statusCode = statusLine.getStatusCode(); if (statusCode == HttpConstants.Codes.SUCCESS) { logger.debug("{} Connection successfully established", name); statsReporter.incrNum200s(); connectionEstablished.set(true); addEvent(new HttpResponseEvent(EventType.CONNECTED, statusLine)); reconnectionManager.resetCounts(); return true; } logger.warn(name + " Error connecting w/ status code - {}, reason - {}", statusCode, statusLine.getReasonPhrase()); statsReporter.incrNumConnectionFailures(); addEvent(new HttpResponseEvent(EventType.HTTP_ERROR, statusLine)); if (HttpConstants.FATAL_CODES.contains(statusCode)) { setExitStatus(new Event(EventType.STOPPED_BY_ERROR, "Fatal error code: " + statusCode)); } else if (statusCode < 500 && statusCode >= 400) { statsReporter.incrNum400s(); // we will retry these a set number of times, then fail if (reconnectionManager.shouldReconnectOn400s()) { logger.debug("{} Reconnecting on {}", name, statusCode); reconnectionManager.handleExponentialBackoff(); } else { logger.debug("{} Reconnecting retries exhausted for {}", name, statusCode); setExitStatus(new Event(EventType.STOPPED_BY_ERROR, "Retries exhausted")); } } else if (statusCode >= 500) { statsReporter.incrNum500s(); reconnectionManager.handleExponentialBackoff(); } else { setExitStatus(new Event(EventType.STOPPED_BY_ERROR, statusLine.getReasonPhrase())); } return false; } private void processConnectionData(Connection conn) { logger.info("{} Processing connection data", name); try { addEvent(new Event(EventType.PROCESSING, "Processing messages")); while (!isDone() && !reconnect.getAndSet(false)) { if (conn.processResponse()) { statsReporter.incrNumMessages(); } else { statsReporter.incrNumMessagesDropped(); } rateTracker.eventObserved(); } } catch (RuntimeException e) { logger.warn(name + " Unknown error processing connection: ", e); statsReporter.incrNumDisconnects(); addEvent(new Event(EventType.DISCONNECTED, e)); } catch (IOException ex) { // connection issue? whatever. let's try connecting again // we can't really diagnosis the actual disconnection reason without parsing (looking at disconnect message) // but we can make a good guess at when we're stalling. TODO logger.info("{} Disconnected during processing - will reconnect", name); statsReporter.incrNumDisconnects(); addEvent(new Event(EventType.DISCONNECTED, ex)); } catch (InterruptedException interrupt) { // interrupted while trying to append message to queue. exit logger.info("{} Thread interrupted during processing, exiting", name); statsReporter.incrNumDisconnects(); setExitStatus(new Event(EventType.STOPPED_BY_ERROR, interrupt)); } catch (Exception e) { // Unexpected exception thrown, killing everything logger.warn(name + " Unexpected exception during processing", e); statsReporter.incrNumDisconnects(); setExitStatus(new Event(EventType.STOPPED_BY_ERROR, e)); } } private void setExitStatus(Event event) { logger.info("{} exit event - {}", name, event.getMessage()); addEvent(event); exitEvent.set(event); } private void addEvent(Event event) { if (eventsQueue != null) { if (!eventsQueue.offer(event)) { statsReporter.incrNumClientEventsDropped(); } } } public void reconnect() { if (connectionEstablished.get()) { reconnect.set(true); } } /** * Stops the current connection. No reconnecting will occur. Kills thread + cleanup. * Waits for the loop to end **/ public void stop(int waitMillis) throws InterruptedException { try { if (!isDone()) { setExitStatus(new Event(EventType.STOPPED_BY_USER, String.format("Stopped by user: waiting for %d ms", waitMillis))); } if (!waitForFinish(waitMillis)) { logger.warn("{} Client thread failed to finish in {} millis", name, waitMillis); } } finally { rateTracker.shutdown(); } } public void shutdown(int millis) { try { stop(millis); } catch (InterruptedException e) { logger.warn("Client failed to shutdown due to interruption", e); } } public boolean isDone() { return exitEvent.get() != null; } public Event getExitEvent() { if (!isDone()) { throw new IllegalStateException(name + " Still running"); } return exitEvent.get(); } public boolean waitForFinish(int millis) throws InterruptedException { return isRunning.await(millis, TimeUnit.MILLISECONDS); } public void waitForFinish() throws InterruptedException { isRunning.await(); } @Override public String toString() { return String.format("%s, endpoint: %s", getName(), endpoint.getURI()); } public String getName() { return name; } public StreamingEndpoint getEndpoint() { return endpoint; } public StatsReporter.StatsTracker getStatsTracker() { return statsReporter.getStatsTracker(); } }