org.aliuge.crawler.pendingqueue.AbstractsPendingQueue.java Source code

Introduction

Here is the source code for org.aliuge.crawler.pendingqueue.AbstractsPendingQueue.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.aliuge.crawler.pendingqueue;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.Serializable;
import java.util.Map;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicLong;

import netscape.javascript.JSObject;

import org.aliuge.crawler.exception.QueueException;
import org.aliuge.crawler.jobconf.PropertyConfigurationHelper;
import org.apache.log4j.Logger;

import com.google.common.collect.Maps;

public abstract class AbstractsPendingQueue<T> implements Serializable {

    private static final long serialVersionUID = 5705584299326194783L;

    private static Logger log = Logger.getLogger(AbstractsPendingQueue.class);

    private ConcurrentLinkedQueue<T> Queue = null;
    /**
     * ??
     */
    private AtomicLong count = new AtomicLong(0L);
    /**
     * ???
     */
    private AtomicLong success = new AtomicLong(0L);
    /**
     * ?
     */
    private AtomicLong failure = new AtomicLong(0);
    /**
     * 
     */
    private AtomicLong ignored = new AtomicLong(0);

    /**
     * 
     */
    protected AbstractsPendingQueue(String jobName) {
        init(jobName);
    }

    /**
     * @desc ?
     */
    @SuppressWarnings({ "unchecked", "rawtypes" })
    private void init(String jobName) {
        File file = new File(new PropertyConfigurationHelper().getString("status.save.path", "status")
                + File.separator + jobName + File.separator + this.getClass().getSimpleName() + ".good");
        if (file.exists()) {
            try {
                FileInputStream fisUrl = new FileInputStream(file);
                ObjectInputStream oisUrl = new ObjectInputStream(fisUrl);
                AbstractsPendingQueue<T> queue = (AbstractsPendingQueue<T>) oisUrl.readObject();
                oisUrl.close();
                fisUrl.close();
                this.Queue = queue.Queue;
                this.failure = queue.failure;
                this.success = queue.success;
                this.count = queue.count;
                this.ignored = queue.ignored;
                log.info("recovery url queue..." + Queue.size());
            } catch (FileNotFoundException e) {
                e.printStackTrace();
            } catch (IOException e) {
                e.printStackTrace();
            } catch (ClassNotFoundException e) {
                e.printStackTrace();
            }
        }
        if (null == Queue)
            Queue = new ConcurrentLinkedQueue();
    }

    /**
     * @param t
     * @desc ?+1
     */
    public boolean addElement(T t) throws QueueException {
        if (null != t) {
            boolean b = Queue.offer(t);
            if (b) {
                count.getAndIncrement();
            }
            return b;
        }
        return false;
    }

    /**
     * ?+1
     * 
     * @param t
     * @param timeout
     *            (MILLISECONDS)
     * @return
     * @throws QueueException
     */
    /*
     * public boolean addElement(T t,int timeout) throws QueueException { if (t
     * != null) { try { boolean b = Queue.offer(t, timeout,
     * TimeUnit.MILLISECONDS); if(b){ count.getAndIncrement(); } return b; }
     * catch (InterruptedException e) { throw new QueueException("??");
     * } } return false; }
     */

    /**
     * @return
     * @desc ??URL
     */
    public T getElementT() throws QueueException {
        T t = Queue.poll();
        if (t != null)
            count.getAndDecrement();
        return t;

    }

    public boolean isEmpty() {
        return Queue.isEmpty();
    }

    /**
     * @param c
     * @return
     * @desc ??+c
     */
    public long processedSuccess(long c) {
        return success.addAndGet(c);
    }

    /**
     * @return
     * @desc ??+1
     */
    public long processedSuccess() {
        return success.incrementAndGet();
    }

    /**
     * @param c
     * @return
     * @desc ?+c
     */
    public long processedFailure(long c) {
        return failure.addAndGet(c);
    }

    /**
     * @return
     * @desc ?+1
     */
    public long processedFailure() {
        return failure.incrementAndGet();
    }

    /**
     * @return
     * @desc 
     */
    public long urlCount() {
        return count.get();
    }

    /**
     * @return
     * @desc ??
     */
    public long success() {
        return success.get();
    }

    /**
     * @param c
     * @return
     * @desc +c
     */
    public long processedIgnored(long c) {
        return ignored.addAndGet(c);
    }

    /**
     * @return
     * @desc +1
     */
    public long processedIgnored() {
        return ignored.incrementAndGet();
    }

    /**
     * @return
     * @desc 
     */
    public long ignored() {
        return ignored.get();
    }

    /**
     * ??
     * 
     * @return
     */
    public Map<String, Object> pendingStatus() {
        Map<String, Object> stat = Maps.newConcurrentMap();
        stat.put("count", new Long(Queue.size()));
        stat.put("urls", Queue.toString());
        stat.put("success", success.get());
        stat.put("failure", failure.get());
        return stat;
    }

    public AtomicLong getCount() {
        return count;
    }

    public void setCount(AtomicLong count) {
        this.count = count;
    }

    public AtomicLong getSuccess() {
        return success;
    }

    public void setSuccess(AtomicLong success) {
        this.success = success;
    }

    public AtomicLong getFailure() {
        return failure;
    }

    public void setFailure(AtomicLong failure) {
        this.failure = failure;
    }

    public AtomicLong getIgnored() {
        return ignored;
    }

    public void setIgnored(AtomicLong ignored) {
        this.ignored = ignored;
    }

    public ConcurrentLinkedQueue<T> getQueue() {
        return Queue;
    }

    public void setQueue(ConcurrentLinkedQueue<T> queue) {
        Queue = queue;
    }

}