Java tutorial
/* * gnizr is a trademark of Image Matters LLC in the United States. * * The contents of this file are subject to the Mozilla Public License Version * 1.1 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.mozilla.org/MPL/ * * Software distributed under the License is distributed on an "AS IS" basis, * WITHOUT WARRANTY OF ANY KIND, either expressed or implied. See the License * for the specific language governing rights and limitations under the License. * * The Initial Contributor of the Original Code is Image Matters LLC. * Portions created by the Initial Contributor are Copyright (C) 2007 * Image Matters LLC. All Rights Reserved. */ package com.gnizr.core.robot.rss; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.TimerTask; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import org.apache.log4j.Logger; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import com.gnizr.core.bookmark.BookmarkManager; import com.gnizr.core.feed.FeedSubscriptionManager; import com.gnizr.db.dao.Bookmark; import com.gnizr.db.dao.FeedSubscription; import com.gnizr.db.dao.MachineTag; import com.gnizr.db.dao.PointMarker; public class CrawlRssFeed extends TimerTask { private static final Logger logger = Logger.getLogger(CrawlRssFeed.class); private FeedSubscriptionManager feedSubscriptionManager; private BookmarkManager bookmarkManager; private ThreadPoolTaskExecutor threadPoolTaskExecutor; private FeedCrawlerFactory crawlerFactory; private ExecutorService saveBookmarkEntriesExecutor; private boolean serviceEnabled; private int ageHour; public CrawlRssFeed() { saveBookmarkEntriesExecutor = Executors.newSingleThreadExecutor(); serviceEnabled = false; ageHour = 2; } public FeedCrawlerFactory getCrawlerFactory() { return crawlerFactory; } public void setCrawlerFactory(FeedCrawlerFactory crawlerFactory) { this.crawlerFactory = crawlerFactory; } public BookmarkManager getBookmarkManager() { return bookmarkManager; } public void setBookmarkManager(BookmarkManager bookmarkManager) { this.bookmarkManager = bookmarkManager; } public FeedSubscriptionManager getFeedSubscriptionManager() { return feedSubscriptionManager; } public void setFeedSubscriptionManager(FeedSubscriptionManager feedSubscriptionManager) { this.feedSubscriptionManager = feedSubscriptionManager; } public ThreadPoolTaskExecutor getThreadPoolTaskExecutor() { return threadPoolTaskExecutor; } public void setThreadPoolTaskExecutor(ThreadPoolTaskExecutor taskExecutor) { this.threadPoolTaskExecutor = taskExecutor; } public void shutdown() { try { System.out.println("CrawlRssFeed shutdown."); List<Runnable> activeTasks = saveBookmarkEntriesExecutor.shutdownNow(); System.out.println("After shutdown. Number of active tasks = " + activeTasks.size()); if (activeTasks.size() > 0) { System.out.println("calling Thread.interrputed"); Thread.interrupted(); System.out.println("done calling Thread.interrputed"); } } catch (Exception e) { } if (threadPoolTaskExecutor != null) { try { threadPoolTaskExecutor.shutdown(); } catch (Exception e) { } } } public void awaitAndShutdown(long time, TimeUnit timeUnit) { if (saveBookmarkEntriesExecutor != null) { try { saveBookmarkEntriesExecutor.awaitTermination(time, TimeUnit.SECONDS); } catch (InterruptedException e) { logger.error("error shutdown saveBookmarkEntriesExecutor.", e); } } } @Override public void run() { logger.debug("CrawlRssFeed run() starts."); if (threadPoolTaskExecutor == null) { logger.error("ThreadPoolTaskExecutor is not defined."); return; } int curActiveThread = threadPoolTaskExecutor.getActiveCount(); boolean isInitOkay = isInitialized(); logger.info("Total number of active TaskExecutor thread: " + curActiveThread); logger.info("Is initialized properly: " + isInitOkay); logger.info("Is service enabled: " + serviceEnabled); if (isInitOkay == true && serviceEnabled == true && curActiveThread == 0) { logger.info("Crawling starts... ThreadId=" + Thread.currentThread().getId()); List<FeedSubscription> feeds = feedSubscriptionManager.listAutoImportSubscription(getAgeHour()); logger.info("Total number of subscriptions to process: " + feeds.size()); FeedCrawlerFactory factory = getDefaultFactoryIfNull(); for (FeedSubscription aFeed : feeds) { FeedCrawler crawler = factory.createFeedCrawler(); threadPoolTaskExecutor.execute(new FeedCrawlerRunnable(crawler, aFeed)); } logger.info("Crawling ends... ThreadId=" + Thread.currentThread().getId()); } else if (serviceEnabled == false) { logger.debug("CrawlRSSFeed service is currently turned off."); } else { logger.info("Skip this run of CrawlRssFeed."); } logger.debug("CrawlRssFeed run() ends."); } public boolean isServiceEnabled() { return serviceEnabled; } public void setServiceEnabled(boolean serviceEnabled) { this.serviceEnabled = serviceEnabled; logger.debug("CrawlRSSFeed setServiceEnabled: " + serviceEnabled); } private void processDoCrawlResult(FeedCrawlResult result, FeedSubscription srcFeed) { try { saveBookmarkEntriesExecutor.execute(new SaveBookmarkEntryRunnable(result, srcFeed)); } catch (Exception e) { logger.error("can't submit new tasks to saveBookmarkEntriesExecutor", e); } } private class SaveBookmarkEntryRunnable implements Runnable { private List<BookmarkEntry> entries; private FeedSubscription srcFeed; private Date feedLastUpdated; private Date feedPubDate; public SaveBookmarkEntryRunnable(FeedCrawlResult result, FeedSubscription srcFeed) { if (result.getEntries() != null) { this.entries = result.getEntries(); } else { this.entries = new ArrayList<BookmarkEntry>(); } this.feedLastUpdated = result.getFeedLastUpdated(); this.feedPubDate = result.getFeedPubDate(); this.srcFeed = srcFeed; } public void run() { logger.debug("SaveBookmarkEntryRunnable run() starts"); try { boolean doReplace = shouldRepalceOnChange(srcFeed); logger.debug("On Change Do Repalce: " + doReplace + " feed: " + srcFeed.getBookmark().getLink().getUrl()); for (BookmarkEntry entry : entries) { Bookmark bm = entry.getBookmark(); int oldBmId = bookmarkManager.getBookmarkId(bm.getUser(), bm.getLink().getUrl()); if (oldBmId > 0 && doReplace == true) { logger.debug("old bookmark exists."); Bookmark oldBm = bookmarkManager.getBookmark(oldBmId); Date oldBmLastUpdated = oldBm.getLastUpdated(); Date newBmLastUpdated = bm.getLastUpdated(); if (oldBmLastUpdated != null && newBmLastUpdated != null && oldBmLastUpdated.before(newBmLastUpdated) == true) { if (bookmarkManager.deleteBookmark(new Bookmark(oldBmId)) == false) { logger.error("unable to delete bookmark of id: " + oldBmId); continue; } logger.debug("deleted on old bookmark before add it as a new"); oldBmId = 0; } } if (oldBmId <= 0) { logger.debug("adding bookmark: " + bm); int newBmId = bookmarkManager.addBookmark(bm); if (newBmId > 0) { logger.debug("add succeed!"); bm.setId(newBmId); List<PointMarker> pm = entry.getPointMarkers(); if (pm != null && pm.isEmpty() == false) { logger.debug("adding # PointMarkers: " + pm.size()); bookmarkManager.addPointMarkers(bm, pm); } } } } if (feedLastUpdated != null) { FeedSubscription feed = new FeedSubscription(srcFeed); feed.setLastSync(feedLastUpdated); feed.setPubDate(feedPubDate); feedSubscriptionManager.updateSubscription(feed); logger.debug("set FeedSubscription lastUpdate to: " + feedLastUpdated); } else { logger.error("feedLastUpdate is NULL"); } } catch (Exception e) { logger.error("error saving bookmark entries: " + entries, e); } logger.debug("SaveBookmarkEntryRunnable run() ends"); } private boolean shouldRepalceOnChange(FeedSubscription feed) { boolean doReplace = false; List<MachineTag> machineTags = feed.getBookmark().getMachineTagList(); for (MachineTag mt : machineTags) { if (mt.getPredicate().equalsIgnoreCase("onchange") == true && mt.getValue().equalsIgnoreCase("replace")) { doReplace = true; break; } } return doReplace; } } private class FeedCrawlerRunnable implements Runnable { private FeedCrawler crawler; private FeedSubscription feed2crawl; public FeedCrawlerRunnable(FeedCrawler crawler, FeedSubscription feed) { this.crawler = crawler; this.feed2crawl = feed; } public void run() { logger.debug("FeedCrawlerRunnable run() starts..."); FeedCrawlResult result = this.crawler.doCrawl(feed2crawl); processDoCrawlResult(result, feed2crawl); logger.debug("FeedCrawlerRunnable run() ends."); } } private FeedCrawlerFactory getDefaultFactoryIfNull() { if (getCrawlerFactory() == null) { logger.warn("No FeedCrawlerFactory is defined. Use DefaultFeedCrawlerFactory."); crawlerFactory = new DefaultFeedCrawlerFactory(); } return crawlerFactory; } private boolean isInitialized() { if (getBookmarkManager() == null) { logger.error("BookmarkManager is not initialized."); return false; } if (getFeedSubscriptionManager() == null) { logger.error("FeedSubscriptionManager is not initialized."); return false; } if (getThreadPoolTaskExecutor() == null) { logger.error("TaskExecutor is not initialized."); return false; } return true; } public int getAgeHour() { return ageHour; } public void setAgeHour(int ageHour) { this.ageHour = ageHour; } }