com.alibaba.napoli.metamorphosis.client.extension.producer.LocalMessageStorageManager.java Source code

Java tutorial

Introduction

Here is the source code for com.alibaba.napoli.metamorphosis.client.extension.producer.LocalMessageStorageManager.java

Source

/*
 * (C) 2007-2012 Alibaba Group Holding Limited.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * Authors:
 *   wuhua <wq163@163.com> , boyan <killme2008@gmail.com>
 */
package com.alibaba.napoli.metamorphosis.client.extension.producer;

import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.FutureTask;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import com.alibaba.napoli.metamorphosis.client.MetaClientConfig;
import com.alibaba.napoli.common.store.Store;
import com.alibaba.napoli.common.store.journal.JournalStore;
import com.alibaba.napoli.metamorphosis.Message;
import com.alibaba.napoli.metamorphosis.cluster.Partition;
import com.alibaba.napoli.metamorphosis.exception.GetRecoverStorageErrorException;
import com.alibaba.napoli.metamorphosis.exception.UnknowCodecTypeException;
import com.alibaba.napoli.metamorphosis.utils.IdWorker;
import com.alibaba.napoli.metamorphosis.utils.NamedThreadFactory;
import com.alibaba.napoli.metamorphosis.utils.codec.Deserializer;
import com.alibaba.napoli.metamorphosis.utils.codec.Serializer;
import com.alibaba.napoli.metamorphosis.utils.codec.impl.Hessian1Deserializer;
import com.alibaba.napoli.metamorphosis.utils.codec.impl.Hessian1Serializer;
import com.alibaba.napoli.metamorphosis.utils.codec.impl.JavaDeserializer;
import com.alibaba.napoli.metamorphosis.utils.codec.impl.JavaSerializer;

/**
 * ??,recover
 * 
 * @author 
 * @since 2011-8-8 ?10:40:56
 */

public class LocalMessageStorageManager implements MessageRecoverManager {

    protected static final String SPLIT = "@";

    /**
     * topic@partition??store map
     */
    protected final ConcurrentHashMap<String/* topic@partition */, FutureTask<Store>> topicStoreMap = new ConcurrentHashMap<String, FutureTask<Store>>();

    /**
     * ??map
     */
    protected final ConcurrentHashMap<String/* topic@partition */, FutureTask<Boolean>> topicRecoverTaskMap = new ConcurrentHashMap<String, FutureTask<Boolean>>();

    private final Serializer serializer;
    protected final Deserializer deserializer;
    static final Log log = LogFactory.getLog(LocalMessageStorageManager.class);
    private final IdWorker idWorker = new IdWorker(0);

    public static final String DEFAULT_META_LOCALMESSAGE_PATH = System.getProperty("meta.localmessage.path",
            System.getProperty("user.home") + File.separator + ".meta_localmessage");

    public String META_LOCALMESSAGE_PATH;

    private final String META_LOCALMESSAGE_CODEC_TYPE = System.getProperty("meta.localmessage.codec", "java");

    // ???
    protected final ThreadPoolExecutor threadPoolExecutor;

    private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();

    public LocalMessageStorageManager(final MetaClientConfig metaClientConfig) {
        this(metaClientConfig, DEFAULT_META_LOCALMESSAGE_PATH, null);
    }

    public LocalMessageStorageManager(final MetaClientConfig metaClientConfig, final String path,
            final MessageRecoverer messageRecoverer) {
        super();
        this.META_LOCALMESSAGE_PATH = StringUtils.isNotBlank(path) ? path : DEFAULT_META_LOCALMESSAGE_PATH;
        this.messageRecoverer = messageRecoverer;

        if (this.META_LOCALMESSAGE_CODEC_TYPE.equals("java")) {
            this.serializer = new JavaSerializer();
            this.deserializer = new JavaDeserializer();
        } else if (this.META_LOCALMESSAGE_CODEC_TYPE.equals("hessian1")) {
            this.serializer = new Hessian1Serializer();
            this.deserializer = new Hessian1Deserializer();
        } else {
            throw new UnknowCodecTypeException(this.META_LOCALMESSAGE_CODEC_TYPE);
        }

        // ?RecoverThreadCount
        this.threadPoolExecutor = new ThreadPoolExecutor(metaClientConfig.getRecoverThreadCount(),
                metaClientConfig.getRecoverThreadCount(), 60, TimeUnit.SECONDS,
                new ArrayBlockingQueue<Runnable>(100), new NamedThreadFactory("SendRecover-thread"),
                new ThreadPoolExecutor.CallerRunsPolicy());

        this.makeDataDir();
        this.loadStores();

        // ????,??topic???
        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
            @Override
            public void run() {
                log.info("????...");
                LocalMessageStorageManager.this.recover();
            }
        }, 0, metaClientConfig.getRecoverMessageIntervalInMills(), TimeUnit.MILLISECONDS);

    }

    private void loadStores() {
        final File dataPath = new File(this.META_LOCALMESSAGE_PATH);
        final File[] files = dataPath.listFiles();
        for (final File subFile : files) {
            if (subFile.isDirectory()) {
                final String name = subFile.getName();
                final String[] tmps = name.split(SPLIT);
                if (tmps.length != 2) {
                    continue;
                }

                log.info("local message storage " + name + " ...");
                this.getOrCreateStore(tmps[0], new Partition(tmps[1]));
            }
        }
    }

    @Override
    public void recover() {
        final Set<String> names = this.topicStoreMap.keySet();
        if (names == null || names.size() == 0) {
            log.info("SendRecover????");
            return;
        }

        if (this.messageRecoverer != null) {
            for (final String name : names) {
                final String[] tmps = name.split(SPLIT);
                final String topic = tmps[0];
                final Partition partition = new Partition(tmps[1]);
                final int count = this.getMessageCount(topic, partition);
                log.info(name + "????:" + count);
                if (count > 0) {
                    if (!this.recover(topic, partition, this.messageRecoverer)) {
                        log.info("SendRecover?????,????,name=" + name);
                    }
                }
            }
        } else {
            log.warn("messageRecoverer");
        }
    }

    /**
     * ????,?(???????)
     * 
     * @param topic
     * @param partition
     * @param recoverer
     *            ?????
     * @return ?????
     * */
    @Override
    public boolean recover(final String topic, final Partition partition, final MessageRecoverer recoverer) {

        final String name = this.generateKey(topic, partition);
        final FutureTask<Boolean> recoverTask = new FutureTask<Boolean>(new Callable<Boolean>() {

            @Override
            public Boolean call() throws Exception {
                final AtomicLong count = new AtomicLong(0);
                try {

                    final Store store = LocalMessageStorageManager.this.getOrCreateStore(topic, partition);

                    this.innerRecover(store, recoverer, count, name);
                } catch (final Throwable e) {
                    log.error("SendRecover?????,name=" + name, e);
                } finally {
                    log.info("SendRecover????,name=" + name + ",???"
                            + count.get() + "?");
                    LocalMessageStorageManager.this.topicRecoverTaskMap.remove(name);
                }
                return true;
            }

            private void innerRecover(final Store store, final MessageRecoverer recoverer, final AtomicLong count,
                    final String name) throws IOException, Exception {
                final Iterator<byte[]> it = store.iterator();
                while (it.hasNext()) {
                    final byte[] key = it.next();
                    final Message msg = (Message) LocalMessageStorageManager.this.deserializer
                            .decodeObject(store.get(key));
                    recoverer.handle(msg);
                    try {
                        store.remove(key);
                        count.incrementAndGet();
                        if (count.get() % 20000 == 0) {
                            log.info("SendRecover " + name + "????:" + count.get());
                        }
                    } catch (final IOException e) {
                        log.error("SendRecover remove message failed", e);
                    }
                }
            }
        });

        final FutureTask<Boolean> ret = this.topicRecoverTaskMap.putIfAbsent(name, recoverTask);
        if (ret == null) {
            this.threadPoolExecutor.submit(recoverTask);
            return true;
        } else {
            if (log.isDebugEnabled()) {
                log.debug("SendRecover?????,????,name=" + name);
            }
            return false;
        }

    }

    protected Store getOrCreateStore(final String topic, final Partition partition) {
        // topicRandomPartitonstore
        this.getOrCreateStore0(topic, Partition.RandomPartiton);
        return this.getOrCreateStore0(topic, partition);
    }

    private Store getOrCreateStore0(final String topic, final Partition partition) {
        final String name = this.generateKey(topic, partition);
        FutureTask<Store> task = this.topicStoreMap.get(name);
        if (task != null) {
            return this.getStore(name, task);
        } else {
            task = new FutureTask<Store>(new Callable<Store>() {

                @Override
                public Store call() throws Exception {
                    final File file = new File(
                            LocalMessageStorageManager.this.META_LOCALMESSAGE_PATH + File.separator + name);
                    if (!file.exists()) {
                        file.mkdir();
                    }
                    return this.newStore(name);
                }

                private Store newStore(final String name) throws IOException {
                    return LocalMessageStorageManager.this.newStore(name);
                }

            });
            FutureTask<Store> existsTask = this.topicStoreMap.putIfAbsent(name, task);
            if (existsTask == null) {
                task.run();
                existsTask = task;
            }
            return this.getStore(name, existsTask);
        }
    }

    private Store getStore(final String topic, final FutureTask<Store> task) {
        try {
            return task.get();
        } catch (final Throwable t) {
            log.error("?topic=" + topic + "store", t);
            throw new GetRecoverStorageErrorException("?topic=" + topic + "store", t);
        }

    }

    private void makeDataDir() {
        final File file = new File(this.META_LOCALMESSAGE_PATH);
        if (!file.exists()) {
            file.mkdir();
        }
    }

    @Override
    public void shutdown() {
        for (final Map.Entry<String, FutureTask<Store>> entry : this.topicStoreMap.entrySet()) {
            final String name = entry.getKey();
            final FutureTask<Store> task = entry.getValue();
            final Store store = this.getStore(name, task);
            try {
                store.close();
            } catch (final IOException e) {
                // ignore
            }
        }

    }

    @Override
    public void append(final Message message, final Partition partition) throws IOException {
        final Store store = this.getOrCreateStore(message.getTopic(), partition);
        final ByteBuffer buf = ByteBuffer.allocate(16);
        buf.putLong(this.idWorker.nextId());
        store.add(buf.array(), this.serializer.encodeObject(message));
    }

    @Override
    public int getMessageCount(final String topic, final Partition partition) {
        final String name = this.generateKey(topic, partition);
        final FutureTask<Store> task = this.topicStoreMap.get(name);
        if (task != null) {
            return this.getStore(name, task).size();
        } else {
            return 0;
        }
    }

    protected MessageRecoverer messageRecoverer;

    protected String generateKey(final String topic, final Partition partition) {
        return topic + SPLIT + partition;
    }

    @Override
    synchronized public void setMessageRecoverer(final MessageRecoverer recoverer) {
        if (this.messageRecoverer == null) {
            this.messageRecoverer = recoverer;
        }
    }

    protected Store newStore(final String name) throws IOException {
        return new JournalStore(this.META_LOCALMESSAGE_PATH + File.separator + name, name);
    }

}