com.dianping.apistatic.Job.MovieShowBlockListCrawlerJob.java Source code

Introduction

Here is the source code for com.dianping.apistatic.Job.MovieShowBlockListCrawlerJob.java

Source

/**
 * Project: movie-mapi-static
 * 
 * File Created at 2014-11-22
 * $Id$
 * 
 * Copyright 2010 dianping.com.
 * All rights reserved.
 *
 * This software is the confidential and proprietary information of
 * Dianping Company. ("Confidential Information").  You shall not
 * disclose such Confidential Information and shall use it only in
 * accordance with the terms of the license agreement you entered into
 * with dianping.com.
 */
package com.dianping.apistatic.Job;

import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CountDownLatch;

import lombok.AllArgsConstructor;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;

import com.dianping.apistatic.Constants;
import com.dianping.apistatic.builder.MovieShowBlockListBuilder;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;

/**
 * 
 * @author jin.chen.sh
 *
 */
public class MovieShowBlockListCrawlerJob extends BaseCrawlerJob {
    private Set<Integer> movieShowIds = Sets.newHashSet();
    private Map<Integer, Set<Integer>> shopIdMap;
    private Map<Integer, Set<Integer>> movieIdMap;

    /* (non-Javadoc)
     * @see com.dianping.apistatic.Job.BaseCrawlerJob#execute()
     */
    @Override
    protected void execute() throws Exception {
        movieShowIds.clear();
        shopIdMap = readMapFromFile(Constants.SHOPIDMAP_PATH); // key cityid
        movieIdMap = readMapFromFile(Constants.MOVIEIDMAP_PATH); // key cityid

        if (MapUtils.isEmpty(shopIdMap) || MapUtils.isEmpty(movieIdMap)) {
            log("shopIdMapmovieIdMap", new RuntimeException("shopIdMapmovieIdMap"));
            return;
        }

        List<Integer> cityIds = getCityIds();
        if (CollectionUtils.isEmpty(cityIds)) {
            return;
        }

        int partitionSize = (cityIds.size() > 4 ? cityIds.size() / 4 : cityIds.size());
        List<List<Integer>> cityIdsList = Lists.partition(cityIds, partitionSize);
        CountDownLatch countDownLatch = new CountDownLatch(cityIdsList.size());

        for (List<Integer> list : cityIdsList) {
            MovieShowBlockCrawlerThread thread = new MovieShowBlockCrawlerThread(list, countDownLatch);
            thread.start();
        }

        try {
            countDownLatch.await();
        } catch (Exception e) {
            log("countDownLatch ", e);
            return;
        }

        if (CollectionUtils.isNotEmpty(movieShowIds)) {
            writeObjectAsJsonToFile(Constants.MOVIESHOWIDS_PATH, movieShowIds);
        }
    }

    @AllArgsConstructor
    public class MovieShowBlockCrawlerThread extends Thread {
        private List<Integer> cityIds;

        private CountDownLatch latch;

        @Override
        public void run() {
            if (CollectionUtils.isEmpty(cityIds)) {
                latch.countDown();
                return;
            }
            for (Integer cityId : cityIds) {
                log("movieshowblocklist current city: " + cityId);
                MovieShowBlockListBuilder movieShowBlockBuilder = new MovieShowBlockListBuilder(cityId,
                        shopIdMap.get(cityId), movieIdMap.get(cityId));
                movieShowBlockBuilder.buildCache(movieShowBlockBuilder.getUrlAndHeader());
                appendMovieShowIds(movieShowBlockBuilder.getMovieShowIds());
            }
            latch.countDown();
        }

        private synchronized void appendMovieShowIds(Set<Integer> movieShowIds) {
            movieShowIds.addAll(movieShowIds);
        }
    }

}