Java tutorial
/* * Copyright 2009 Gist, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package mx.bigdata.t4j; import twitter4j.*; import twitter4j.auth.Authorization; import twitter4j.conf.Configuration; import twitter4j.auth.AccessToken; import twitter4j.auth.NullAuthorization; import twitter4j.auth.OAuthSupport; import twitter4j.auth.OAuthAuthorization; import twitter4j.internal.http.HttpClientWrapper; import twitter4j.internal.http.HttpClientWrapperConfiguration; import twitter4j.internal.http.HttpParameter; import twitter4j.internal.util.T4JInternalStringUtil; import twitter4j.internal.http.HttpResponse; import twitter4j.internal.json.DataObjectFactoryUtil; import java.net.SocketTimeoutException; import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InterruptedIOException; import java.io.EOFException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; import static twitter4j.internal.http.HttpResponseCode.FORBIDDEN; import static twitter4j.internal.http.HttpResponseCode.NOT_ACCEPTABLE; /* http://apiwiki.twitter.com/Streaming-API-Documentation#Connecting When a network error (TCP/IP level) is encountered, back off linearly. Perhaps start at 250 milliseconds, double, and cap at 16 seconds When a HTTP error (> 200) is returned, back off exponentially. Perhaps start with a 10 second wait, double on each subsequent failure, and finally cap the wait at 240 seconds. Consider sending an alert to a human operator after multiple HTTP errors, as there is probably a client configuration issue that is unlikely to be resolved without human intervention. There's not much point in polling any faster in the face of HTTP error codes and your client is may run afoul of a rate limit. */ class TwitterStreamConsumerImpl extends Thread implements TwitterStreamConsumer { private final static Logger logger = Logger.getLogger(TwitterStreamConsumerImpl.class.getName()); private final BackOff tcpBackOff = new BackOff(true, 250, 16000); private final BackOff httpBackOff = new BackOff(10000, 240000); private final StreamListener listener; private final StatusStreamProvider provider; TwitterStreamConsumerImpl(StatusStreamProvider provider, StreamListener listener) { this.listener = listener; this.provider = provider; } /** * Connects to twitter and processes the streams. On * exception, backs off and reconnects. Runs until the thread * is interrupted. */ //@Override public void run() { logger.info("Begin " + Thread.currentThread().getName()); try { while (true) { if (Thread.currentThread().isInterrupted()) { logger.info("Interrupted " + Thread.currentThread().getName()); return; } try { connectAndProcess(); } catch (SocketTimeoutException ex) { // Handle like an IOException even though it's // an InterruptedIOException. logger.log(Level.WARNING, "Error fetching from " + ex); tcpBackOff.backOff(); } catch (InterruptedException ex) { // Don't let this be handled as a generic Exception. return; } catch (InterruptedIOException ex) { return; } catch (TwitterException ex) { logger.log(Level.WARNING, "Error fetching from " + ex); if (ex.getStatusCode() > 200) { httpBackOff.backOff(); } } catch (IOException ex) { logger.log(Level.WARNING, "Error fetching from " + ex); tcpBackOff.backOff(); } catch (Exception ex) { // This could be a NumberFormatException or // something. Open a new connection to // resync. logger.log(Level.WARNING, "Error fetching from " + ex); } } } catch (InterruptedException ex) { return; } finally { logger.info("End " + Thread.currentThread().getName()); } } /** * Connects to twitter and handles tweets until it gets an * exception or is interrupted. */ private void connectAndProcess() throws InterruptedException, IOException, TwitterException { StatusStream stream = provider.getStream(); try { processStream(stream.getInputStream()); } finally { stream.close(); } } void processStream(InputStream is) throws InterruptedException, IOException { DataInputStream in = new DataInputStream(is); while (true) { if (Thread.currentThread().isInterrupted()) { logger.info("Interrupted PS " + Thread.currentThread().getName()); return; } byte[] bl = new byte[10]; if (readLine(in, bl, 0, bl.length) == -1) { throw new EOFException(); } String lengthBytes = new String(bl).trim(); if (lengthBytes.length() > 0) { byte[] bytes = new byte[Integer.parseInt(lengthBytes)]; in.readFully(bytes); try { listener.onData(bytes); } catch (Throwable caught) { logger.log(Level.SEVERE, caught.toString()); } } } } /* * Originally found in apache-tomcat-6.0.26 * org.apache.tomcat.util.net.TcpConnection * Licensed under the Apache License, Version 2.0 */ private int readLine(InputStream in, byte[] b, int off, int len) throws IOException { if (len <= 0) { return 0; } int count = 0, c; while ((c = in.read()) != -1) { b[off++] = (byte) c; count++; if (c == '\n' || count == len) { break; } } return count > 0 ? count : -1; } /** * Handles backing off for an initial time, doubling until a cap * is reached. */ private static final class BackOff { private final boolean noInitialBackoff; private final long initialMillis; private final long capMillis; private long backOffMillis; /** * @param noInitialBackoff true if the initial backoff should be zero * @param initialMillis the initial amount of time to back off, after * an optional zero-length initial backoff * @param capMillis upper limit to the back off time */ public BackOff(boolean noInitialBackoff, long initialMillis, long capMillis) { this.noInitialBackoff = noInitialBackoff; this.initialMillis = initialMillis; this.capMillis = capMillis; reset(); } public BackOff(long initialMillis, long capMillis) { this(false, initialMillis, capMillis); } public void reset() { if (noInitialBackoff) { backOffMillis = 0; } else { backOffMillis = initialMillis; } } public void backOff() throws InterruptedException { if (backOffMillis == 0) { backOffMillis = initialMillis; } else { Thread.sleep(backOffMillis); backOffMillis *= 2; if (backOffMillis > capMillis) { backOffMillis = capMillis; } } } } }