Java tutorial
/* Copyright 2012 Tim Garrett, Mothsoft LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.mothsoft.alexis.engine.retrieval; import java.net.URL; import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.List; import org.apache.log4j.Logger; import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.support.TransactionCallback; import org.springframework.transaction.support.TransactionCallbackWithoutResult; import org.springframework.transaction.support.TransactionTemplate; import org.springframework.util.CollectionUtils; import twitter4j.HashtagEntity; import twitter4j.Status; import twitter4j.URLEntity; import twitter4j.UserMentionEntity; import twitter4j.auth.AccessToken; import com.mothsoft.alexis.dao.DocumentDao; import com.mothsoft.alexis.dao.SourceDao; import com.mothsoft.alexis.dao.TweetDao; import com.mothsoft.alexis.dao.UserDao; import com.mothsoft.alexis.domain.DocumentUser; import com.mothsoft.alexis.domain.SocialConnection; import com.mothsoft.alexis.domain.Source; import com.mothsoft.alexis.domain.Tweet; import com.mothsoft.alexis.domain.TweetHashtag; import com.mothsoft.alexis.domain.TweetLink; import com.mothsoft.alexis.domain.TweetMention; import com.mothsoft.alexis.domain.TwitterSource; import com.mothsoft.alexis.domain.User; import com.mothsoft.alexis.security.CurrentUserUtil; import com.mothsoft.integration.twitter.TwitterService; public class TwitterRetrievalTaskImpl implements RetrievalTask { private static final Logger logger = Logger.getLogger(TwitterRetrievalTaskImpl.class); private DocumentDao documentDao; private SourceDao sourceDao; private PlatformTransactionManager transactionManager; private TransactionTemplate transactionTemplate; private TweetDao tweetDao; private TwitterService twitterService; private UserDao userDao; private IntelligentDelay delay; public TwitterRetrievalTaskImpl() { delay = new IntelligentDelay("Twitter Retrieval", 5, 90); } public void setDocumentDao(final DocumentDao documentDao) { this.documentDao = documentDao; } public void setSourceDao(final SourceDao sourceDao) { this.sourceDao = sourceDao; } public void setTransactionManager(final PlatformTransactionManager transactionManager) { this.transactionManager = transactionManager; this.transactionTemplate = new TransactionTemplate(this.transactionManager); } public void setTweetDao(final TweetDao tweetDao) { this.tweetDao = tweetDao; } public void setTwitterService(final TwitterService twitterService) { this.twitterService = twitterService; } public void setUserDao(final UserDao userDao) { this.userDao = userDao; } public void retrieve() { CurrentUserUtil.setSystemUserAuthentication(); try { logger.info("Retrieving content from Twitter"); final List<Source> sources = findSourcesToProcess(); for (final Source source : sources) { final TwitterSource twitterSource = (TwitterSource) source; logger.info("Retrieving Twitter source: " + twitterSource.getId() + " using Twitter user: " + twitterSource.getSocialConnection().getRemoteUsername()); handleSource(twitterSource); } if (CollectionUtils.isEmpty(sources)) { logger.info("Twitter retrieval found nothing to do, will sleep"); this.delay.sleep(); } } catch (final Exception e) { logger.error("Exception will cause falloff: " + e, e); this.delay.sleep(); } finally { CurrentUserUtil.clearAuthentication(); } } private List<Source> findSourcesToProcess() { try { return this.transactionTemplate.execute(new TransactionCallback<List<Source>>() { public List<Source> doInTransaction(TransactionStatus status) { return TwitterRetrievalTaskImpl.this.sourceDao .listSourcesWithRetrievalDateMoreThanXMinutesAgo(15, TwitterSource.class); } }); } catch (final Exception e) { logger.error("Listing sources for retrieval failed: " + e, e); return Collections.emptyList(); } } private void handleSource(final TwitterSource twitterSource) { try { this.transactionTemplate.execute(new TransactionCallbackWithoutResult() { @Override protected void doInTransactionWithoutResult(TransactionStatus status) { handleSourceImpl(twitterSource); } }); } catch (final Exception e) { throw new RuntimeException(e); } } private void handleSourceImpl(final TwitterSource twitterSource) { final SocialConnection socialConnection = twitterSource.getSocialConnection(); final AccessToken accessToken = new AccessToken(socialConnection.getOauthToken(), socialConnection.getOauthTokenSecret()); final List<Status> statuses = this.twitterService.getHomeTimeline(accessToken, twitterSource.getLastTweetId(), (short) 800); if (statuses != null && statuses.size() > 0) { logger.info("Twitter retrieval found " + statuses.size() + " Tweets for user: " + socialConnection.getRemoteUsername()); // the newest tweet in the timeline will be our starting point for // the next fetch twitterSource.setLastTweetId(statuses.get(0).getId()); // import these in reverse order to ensure newest ones have the // highest document IDs Collections.reverse(statuses); final Long userId = twitterSource.getUserId(); final User user = this.userDao.get(userId); for (final Status status : statuses) { final Long tweetId = status.getId(); Tweet tweet = this.tweetDao.getTweetByRemoteTweetId(tweetId); final boolean isAdd = (tweet == null); if (isAdd) { // TODO - is this right? // Twitter allows 2 different styles of retweets. The // ones that are actually retweets show as tweeted by the // original user. Others may show // "RT @original thing original said" tweeted // by the new person final boolean retweet = status.isRetweet(); final twitter4j.User tweeter; final String text; twitter4j.User retweeter = null; final List<TweetLink> links; final List<TweetMention> mentions; final List<TweetHashtag> hashtags; if (retweet) { tweeter = status.getRetweetedStatus().getUser(); text = status.getRetweetedStatus().getText(); retweeter = status.getUser(); links = readLinks(status.getRetweetedStatus()); mentions = readMentions(status.getRetweetedStatus()); hashtags = readHashtags(status.getRetweetedStatus()); } else { tweeter = status.getUser(); text = status.getText(); links = readLinks(status); mentions = readMentions(status); hashtags = readHashtags(status); } final URL profileImageUrl = tweeter.getProfileImageUrlHttps(); final Date createdAt = status.getCreatedAt(); tweet = new Tweet(tweetId, createdAt, tweeter.getScreenName(), tweeter.getName(), profileImageUrl, text, links, mentions, hashtags, retweet, retweet ? retweeter.getScreenName() : null); this.documentDao.add(tweet); } final DocumentUser documentUser = new DocumentUser(tweet, user); if (isAdd || !tweet.getDocumentUsers().contains(documentUser)) { tweet.getDocumentUsers().add(new DocumentUser(tweet, user)); this.documentDao.update(tweet); } } } else { logger.info("Twitter retrieval found no Tweets for user: " + socialConnection.getRemoteUsername()); } twitterSource.setRetrievalDate(new Date()); this.sourceDao.update(twitterSource); } private List<TweetLink> readLinks(Status status) { final List<TweetLink> links = new ArrayList<TweetLink>(); if (status.getURLEntities() != null) { for (final URLEntity entity : status.getURLEntities()) { final String displayUrl = entity.getDisplayURL(); final String expandedUrl = entity.getExpandedURL(); final String url = entity.getURL(); final TweetLink link = new TweetLink((short) entity.getStart(), (short) entity.getEnd(), displayUrl, expandedUrl, url); links.add(link); } } return links; } private List<TweetMention> readMentions(Status status) { final List<TweetMention> mentions = new ArrayList<TweetMention>(); if (status.getUserMentionEntities() != null) { for (final UserMentionEntity entity : status.getUserMentionEntities()) { final Long userId = entity.getId(); final String name = entity.getName(); final String screenName = entity.getScreenName(); final TweetMention mention = new TweetMention((short) entity.getStart(), (short) entity.getEnd(), userId, name, screenName); mentions.add(mention); } } return mentions; } private List<TweetHashtag> readHashtags(Status status) { final List<TweetHashtag> hashtags = new ArrayList<TweetHashtag>(); if (status.getHashtagEntities() != null) { for (final HashtagEntity entity : status.getHashtagEntities()) { final TweetHashtag hashtag = new TweetHashtag((short) entity.getStart(), (short) entity.getEnd(), entity.getText()); hashtags.add(hashtag); } } return hashtags; } }