org.sbs.goodcrawler.fetcher.FailedPageBackup.java Source code

Introduction

Here is the source code for org.sbs.goodcrawler.fetcher.FailedPageBackup.java

Source

/**
 * ########################  SHENBAISE'S WORK  ##########################
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sbs.goodcrawler.fetcher;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.sbs.goodcrawler.conf.GlobalConstants;
import org.sbs.goodcrawler.conf.PropertyConfigurationHelper;
import org.sbs.goodcrawler.conf.Worker;
import org.sbs.goodcrawler.exception.QueueException;
import org.sbs.goodcrawler.page.Page;
import org.sbs.util.DateTimeUtil;

/**
 * @author whiteme
 * @date 2013730
 * @desc ?
 */
public class FailedPageBackup {

    private Log log = LogFactory.getLog(this.getClass());
    private static FailedPageBackup instance = null;
    private PropertyConfigurationHelper config = PropertyConfigurationHelper.getInstance();
    /**
     * ??
     */
    private BlockingQueue<Page> Queue = null;
    /**
     * ???????
     */
    private boolean ignoreFailedPage = true;

    private FailedPageBackup() {
        init();
    }

    public static FailedPageBackup getInstace() {
        if (null == instance) {
            instance = new FailedPageBackup();
        }
        return instance;
    }

    public void init() {
        ignoreFailedPage = Boolean.getBoolean(config.getString(GlobalConstants.ignoreFailedPages, "true"));
        if (!ignoreFailedPage) {
            Queue = new LinkedBlockingDeque<Page>(config.getInt(GlobalConstants.failedPagesQueueSize, 2000));
            // 
            BackupFailedPages backup = new BackupFailedPages();
            Thread failedPagesBackupThread = new Thread(backup, "failed-pages-backup-thread");
            ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
            scheduler.scheduleAtFixedRate(failedPagesBackupThread, 60, 60, TimeUnit.SECONDS);
        }
    }

    /**
     * ??
     * 
     * @param page
     * @desc
     */
    public void addPage(Page page) throws QueueException {
        if (page != null) {
            try {
                Queue.put(page);
            } catch (InterruptedException e) {
                throw new QueueException("???");
            }
        }
    }

    /**
     * ??
     * 
     * @return
     * @desc
     */
    public Page getPage() throws QueueException {
        try {
            return Queue.take();
        } catch (InterruptedException e) {
            throw new QueueException("????");
        }
    }

    public boolean isEmpty() {
        return Queue.isEmpty();
    }

    /**
     * @author shenbaise(shenbaise@outlook.com)
     * @date 2013-6-30 ?
     */
    private class BackupFailedPages implements Runnable {
        @Override
        public void run() {
            Page page;
            boolean flag = true;
            File backFile = null;
            FileChannel fc = null;
            byte[] b = new byte[] { (byte) 1, (byte) 1 };
            if (Worker.isStop())
                if (!ignoreFailedPage) {
                    backFile = new File(config.getString(GlobalConstants.failedPagesBackupPath, "")
                            + File.pathSeparator + DateTimeUtil.getDate());
                    try {
                        fc = new FileOutputStream(backFile, true).getChannel();
                        if (flag) {
                            while (null != (page = Queue.poll())) {
                                fc.write(ByteBuffer.wrap(page.getContentData()));
                                fc.write(ByteBuffer.wrap(b));
                            }
                            fc.close();
                        }
                    } catch (IOException e) {
                        e.printStackTrace();
                        log.warn(e.getMessage());
                    }
                }
        }
    }
}