com.dianping.apistatic.Job.MovieShowDetailCrawlerJob.java Source code

Introduction

Here is the source code for com.dianping.apistatic.Job.MovieShowDetailCrawlerJob.java

Source

/**
 * Project: movie-mapi-static
 * 
 * File Created at 2014-11-22
 * $Id$
 * 
 * Copyright 2010 dianping.com.
 * All rights reserved.
 *
 * This software is the confidential and proprietary information of
 * Dianping Company. ("Confidential Information").  You shall not
 * disclose such Confidential Information and shall use it only in
 * accordance with the terms of the license agreement you entered into
 * with dianping.com.
 */
package com.dianping.apistatic.Job;

import java.util.List;
import java.util.concurrent.CountDownLatch;

import lombok.AllArgsConstructor;

import org.apache.commons.collections.CollectionUtils;

import com.dianping.apistatic.Constants;
import com.dianping.apistatic.builder.MovieShowBuilder;
import com.google.common.collect.Lists;

/**
 * 
 * @author jin.chen.sh
 *
 */
public class MovieShowDetailCrawlerJob extends BaseCrawlerJob {

    /* (non-Javadoc)
     * @see com.dianping.apistatic.Job.BaseCrawlerJob#execute()
     */
    @Override
    protected void execute() throws Exception {
        List<Integer> movieShowIds = readIntListFromFile(Constants.MOVIESHOWIDS_PATH);
        if (CollectionUtils.isEmpty(movieShowIds)) {
            log("movieShowIds?", new RuntimeException("movieShowIds "));
            return;
        }

        int partitionSize = (movieShowIds.size() > 5 ? movieShowIds.size() / 5 : movieShowIds.size());
        List<List<Integer>> movieShowIdsList = Lists.partition(movieShowIds, partitionSize);
        CountDownLatch countDownLatch = new CountDownLatch(movieShowIdsList.size());

        for (List<Integer> list : movieShowIdsList) {
            MovieShowCrawlerThread thread = new MovieShowCrawlerThread(list, countDownLatch);
            thread.start();
        }
        try {
            countDownLatch.await();
        } catch (InterruptedException e) {
            log("CountDownLatch InterruptedException", e);
        }
    }

    @AllArgsConstructor
    public class MovieShowCrawlerThread extends Thread {

        private List<Integer> movieShowIds;

        private CountDownLatch latch;

        @Override
        public void run() {
            if (CollectionUtils.isEmpty(movieShowIds)) {
                latch.countDown();
                return;
            }
            for (Integer movieShowId : movieShowIds) {
                MovieShowBuilder movieShowBuilder = new MovieShowBuilder(movieShowId);
                movieShowBuilder.buildCache(movieShowBuilder.getUrlAndHeader());
            }
            latch.countDown();
        }
    }

}