Java tutorial
/* * (C) 2007-2012 Alibaba Group Holding Limited. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * Authors: * dogun (yuexuqiang at gmail.com) */ package com.taobao.common.store.journal; import java.io.File; import java.io.FileNotFoundException; import java.io.FilenameFilter; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Calendar; import java.util.Date; import java.util.GregorianCalendar; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.log4j.Logger; import com.taobao.common.store.Store; import com.taobao.common.store.journal.impl.ConcurrentIndexMap; import com.taobao.common.store.journal.impl.LRUIndexMap; import com.taobao.common.store.util.BytesKey; import com.taobao.common.store.util.Util; /** * <b>key/value</b> * * key16 <br /> * 1<br /> * name.1 name.1.log<br /> * 2data<br /> * 3log+key+<br /> * 4name.1offsetlength<br /> * 5<br /> * 6<br /> * 7<br /> * 8log<br /> * * @author dogun (yuexuqiang at gmail.com) */ public class JournalStore implements Store, JournalStoreMBean { private final Log log = LogFactory.getLog(JournalStore.class); public static final int FILE_SIZE = 1024 * 1024 * 64; // 20M // public static final int ONE_DAY = 1000 * 60 * 60 * 24; public static final int HALF_DAY = 1000 * 60 * 60 * 12; protected static final int DEFAULT_MAX_BATCH_SIZE = 1024 * 1024 * 4; private final String path; private final String name; private final boolean force; protected IndexMap indices; private final Map<BytesKey, Long> lastModifiedMap = new ConcurrentHashMap<BytesKey, Long>(); public Map<Integer, DataFile> dataFiles = new ConcurrentHashMap<Integer, DataFile>(); protected Map<Integer, LogFile> logFiles = new ConcurrentHashMap<Integer, LogFile>(); protected DataFile dataFile = null; protected LogFile logFile = null; private DataFileAppender dataFileAppender = null; private final AtomicInteger number = new AtomicInteger(0); private long intervalForCompact = HALF_DAY; private long intervalForRemove = HALF_DAY * 2 * 7; private volatile ScheduledExecutorService scheduledPool; private volatile long maxFileCount = Long.MAX_VALUE; protected int maxWriteBatchSize = DEFAULT_MAX_BATCH_SIZE; public static class InflyWriteData { public volatile byte[] data; public volatile int count; public InflyWriteData(final byte[] data) { super(); this.data = data; this.count = 1; } } /** * pathname * * @param path * @param name * @param force * @throws IOException */ public JournalStore(final String path, final String name, final boolean force, final boolean enabledIndexLRU) throws IOException { this(path, name, null, force, enabledIndexLRU, false); } /** * * * @param path * @param name * @param indices * @param force * @param enabledIndexLRU * @throws IOException */ public JournalStore(final String path, final String name, final IndexMap indices, final boolean force, final boolean enabledIndexLRU) throws IOException { this(path, name, indices, force, enabledIndexLRU, false); } /** * * @param path * @param name * @param force * @param enableIndexLRU * @param enabledDataFileCheck * @throws IOException */ public JournalStore(final String path, final String name, final boolean force, final boolean enableIndexLRU, final boolean enabledDataFileCheck) throws IOException { this(path, name, null, force, enableIndexLRU, false); } /** * * * @param path * @param name * @param force * @throws IOException */ public JournalStore(final String path, final String name, final IndexMap indices, final boolean force, final boolean enableIndexLRU, final boolean enabledDataFileCheck) throws IOException { Util.registMBean(this, name); this.path = path; this.name = name; this.force = force; if (indices == null) { if (enableIndexLRU) { final long maxMemory = Runtime.getRuntime().maxMemory(); // 1/40 final int capacity = (int) (maxMemory / 40 / 60); this.indices = new LRUIndexMap(capacity, this.getPath() + File.separator + name + "_indexCache", enableIndexLRU); } else { this.indices = new ConcurrentIndexMap(); } } else { this.indices = indices; } this.dataFileAppender = new DataFileAppender(this); this.initLoad(); // if (null == this.dataFile || null == this.logFile) { this.newDataFile(); } // Store4j. if (enabledDataFileCheck) { this.scheduledPool = Executors.newSingleThreadScheduledExecutor(); this.scheduledPool.scheduleAtFixedRate(new DataFileCheckThread(), this.calcDelay(), HALF_DAY, TimeUnit.MILLISECONDS); log.warn(""); } // ,,. Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { try { JournalStore.this.close(); } catch (final IOException e) { log.error("close error", e); } } }); } /** * pathname * * @param path * @param name * @throws IOException */ public JournalStore(final String path, final String name) throws IOException { this(path, name, false, false); } /* * (non-Javadoc) * * @see com.taobao.common.store.Store#add(byte[], byte[]) */ @Override public void add(final byte[] key, final byte[] data) throws IOException { this.add(key, data, false); } @Override public void add(final byte[] key, final byte[] data, final boolean force) throws IOException { // name.1offset this.checkParam(key, data); this.innerAdd(key, data, -1, force); } @Override public boolean remove(final byte[] key, final boolean force) throws IOException { return this.innerRemove(key, force); } /** * . * * @param key * @throws IOException */ private void reuse(final byte[] key, final boolean sync) throws IOException { final byte[] value = this.get(key); final long oldLastTime = this.lastModifiedMap.get(new BytesKey(key)); if (value != null && this.remove(key)) { this.innerAdd(key, value, oldLastTime, sync); } } /** * delay. * * @return */ private long calcDelay() { final Calendar date = new GregorianCalendar(); date.setTime(new Date()); final long currentTime = date.getTime().getTime(); date.set(Calendar.HOUR_OF_DAY, 6); date.set(Calendar.MINUTE, 0); date.set(Calendar.SECOND, 0); long delay = date.getTime().getTime() - currentTime; // 66 if (delay < 0) { date.set(Calendar.HOUR_OF_DAY, 18); date.set(Calendar.MINUTE, 0); date.set(Calendar.SECOND, 0); delay = date.getTime().getTime() - currentTime; // 6 if (delay < 0) { delay += HALF_DAY; } } return delay; } /** * * * @param key * @param data * @throws IOException */ private OpItem innerAdd(final byte[] key, final byte[] data, final long oldLastTime, final boolean sync) throws IOException { final BytesKey k = new BytesKey(key); final OpItem op = new OpItem(); op.op = OpItem.OP_ADD; this.dataFileAppender.store(op, k, data, sync); this.indices.put(k, op); if (oldLastTime == -1) { this.lastModifiedMap.put(k, System.currentTimeMillis()); } else { this.lastModifiedMap.put(k, oldLastTime); } return op; } /* * (non-Javadoc) * * @see com.taobao.common.store.Store#get(byte[]) */ @Override public byte[] get(final byte[] key) throws IOException { byte[] data = null; final BytesKey bytesKey = new BytesKey(key); data = this.dataFileAppender.getDataFromInFlyWrites(bytesKey); if (data != null) { return data; } final OpItem op = this.indices.get(bytesKey); if (null != op) { final DataFile df = this.dataFiles.get(Integer.valueOf(op.number)); if (null != df) { final ByteBuffer bf = ByteBuffer.wrap(new byte[op.length]); df.read(bf, op.offset); data = bf.array(); } else { log.warn("" + op); this.indices.remove(bytesKey); this.lastModifiedMap.remove(bytesKey); } } return data; } /* * (non-Javadoc) * * @see com.taobao.common.store.Store#iterator() */ @Override public Iterator<byte[]> iterator() throws IOException { final Iterator<BytesKey> it = this.indices.keyIterator(); return new Iterator<byte[]>() { @Override public boolean hasNext() { return it.hasNext(); } @Override public byte[] next() { final BytesKey bk = it.next(); if (null != bk) { return bk.getData(); } return null; } @Override public void remove() { throw new UnsupportedOperationException("store.remove"); } }; } /* * (non-Javadoc) * * @see com.taobao.common.store.Store#remove(byte[]) */ @Override public boolean remove(final byte[] key) throws IOException { return this.remove(key, false); } /** * * * @param key * @return * @throws IOException */ private boolean innerRemove(final byte[] key, final boolean sync) throws IOException { boolean ret = false; final BytesKey k = new BytesKey(key); final OpItem op = this.indices.get(k); if (null != op) { ret = this.innerRemove(op, k, sync); if (ret) { this.indices.remove(k); this.lastModifiedMap.remove(k); } } return ret; } /** * OpItem. * * @param op * @return * @throws IOException */ private boolean innerRemove(final OpItem op, final BytesKey bytesKey, final boolean sync) throws IOException { final DataFile df = this.dataFiles.get(Integer.valueOf(op.number)); final LogFile lf = this.logFiles.get(Integer.valueOf(op.number)); if (null != df && null != lf) { final OpItem o = new OpItem(); o.key = op.key; o.length = op.length; o.number = op.number; o.offset = op.offset; o.op = OpItem.OP_DEL; this.dataFileAppender.remove(o, bytesKey, sync); return true; } return false; } /** * * * @param key * @param data */ private void checkParam(final byte[] key, final byte[] data) { if (null == key || null == data) { throw new NullPointerException("key/data can't be null"); } if (key.length != 16) { throw new IllegalArgumentException("key.length must be 16"); } } /** * * * @throws FileNotFoundException */ protected DataFile newDataFile() throws IOException { if (this.dataFiles.size() > this.maxFileCount) { throw new RuntimeException("" + this.maxFileCount + ""); } final int n = this.number.incrementAndGet(); this.dataFile = new DataFile(new File(this.path + File.separator + this.name + "." + n), n, this.force); this.logFile = new LogFile(new File(this.path + File.separator + this.name + "." + n + ".log"), n, this.force); this.dataFiles.put(Integer.valueOf(n), this.dataFile); this.logFiles.put(Integer.valueOf(n), this.logFile); log.info("" + this.dataFile); return this.dataFile; } /** * Create the parent directory if it doesn't exist. */ private void checkParentDir(final File parent) { if (!parent.exists() && !parent.mkdirs()) { throw new IllegalStateException("Can't make dir " + this.path); } } /** * * * @throws IOException */ private void initLoad() throws IOException { log.warn(""); final String nm = this.name + "."; final File dir = new File(this.path); this.checkParentDir(dir); final File[] fs = dir.listFiles(new FilenameFilter() { @Override public boolean accept(final File dir, final String n) { return n.startsWith(nm) && !n.endsWith(".log"); } }); if (fs == null || fs.length == 0) { return; } log.warn(""); final List<Integer> indexList = new LinkedList<Integer>(); for (final File f : fs) { try { final String fn = f.getName(); final int n = Integer.parseInt(fn.substring(nm.length())); indexList.add(Integer.valueOf(n)); } catch (final Exception e) { log.error("parse file index error" + f, e); } } Integer[] indices = indexList.toArray(new Integer[indexList.size()]); // Arrays.sort(indices); for (final Integer n : indices) { log.warn("index" + n + ""); // final Map<BytesKey, OpItem> idx = new HashMap<BytesKey, OpItem>(); // dataFilelogFile final File f = new File(dir, this.name + "." + n); final DataFile df = new DataFile(f, n, this.force); final LogFile lf = new LogFile(new File(f.getAbsolutePath() + ".log"), n, this.force); final long size = lf.getLength() / OpItem.LENGTH; for (int i = 0; i < size; ++i) { // final ByteBuffer bf = ByteBuffer.wrap(new byte[OpItem.LENGTH]); lf.read(bf, i * OpItem.LENGTH); if (bf.hasRemaining()) { log.warn("log file error:" + lf + ", index:" + i); continue; } final OpItem op = new OpItem(); op.parse(bf.array()); final BytesKey key = new BytesKey(op.key); switch (op.op) { case OpItem.OP_ADD: // final OpItem o = this.indices.get(key); if (null != o) { // UpdateRemove // Remove this.innerRemove(o, key, true); // map this.indices.remove(key); this.lastModifiedMap.remove(key); } boolean addRefCount = true; if (idx.get(key) != null) { // addupdate addRefCount = false; } idx.put(key, op); if (addRefCount) { df.increment(); } break; case OpItem.OP_DEL: // idx.remove(key); df.decrement(); break; default: log.warn("unknow op:" + (int) op.op); break; } } if (df.getLength() >= FILE_SIZE && df.isUnUsed()) { // df.delete(); lf.delete(); log.warn(""); } else { // map this.dataFiles.put(n, df); this.logFiles.put(n, lf); if (!df.isUnUsed()) { // this.indices.putAll(idx); // ,. final long lastModified = lf.lastModified(); for (final BytesKey key : idx.keySet()) { this.lastModifiedMap.put(key, lastModified); } log.warn("referenceCount:" + df.getReferenceCount() + ", index:" + idx.size()); } } } // if (this.dataFiles.size() > 0) { indices = this.dataFiles.keySet().toArray(new Integer[this.dataFiles.keySet().size()]); Arrays.sort(indices); for (int i = 0; i < indices.length - 1; i++) { final DataFile df = this.dataFiles.get(indices[i]); if (df.isUnUsed() || df.getLength() < FILE_SIZE) { throw new IllegalStateException("used"); } } final Integer n = indices[indices.length - 1]; this.number.set(n.intValue()); this.dataFile = this.dataFiles.get(n); this.logFile = this.logFiles.get(n); } log.warn("" + this.size()); } /* * (non-Javadoc) * * @see com.taobao.common.store.Store#size() */ @Override public int size() { return this.indices.size(); } /* * (non-Javadoc) * * @see com.taobao.common.store.Store#update(byte[], byte[]) */ @Override public boolean update(final byte[] key, final byte[] data) throws IOException { // UpdateOpCodeUpdate final BytesKey k = new BytesKey(key); final OpItem op = this.indices.get(k); if (null != op) { this.indices.remove(k); final OpItem o = this.innerAdd(key, data, -1, false); if (o.number != op.number) { // this.innerRemove(op, k, false); } else { // DataFileadd final DataFile df = this.dataFiles.get(Integer.valueOf(op.number)); df.decrement(); } return true; } return false; } /* * (non-Javadoc) * * @see com.taobao.common.store.journal.JournalStoreMBean#getDataFilesInfo() */ @Override public String getDataFilesInfo() { return this.dataFiles.toString(); } /* * (non-Javadoc) * * @see com.taobao.common.store.journal.JournalStoreMBean#getLogFilesInfo() */ @Override public String getLogFilesInfo() { return this.logFiles.toString(); } /* * (non-Javadoc) * * @see com.taobao.common.store.journal.JournalStoreMBean#getNumber() */ @Override public int getNumber() { return this.number.get(); } /* * (non-Javadoc) * * @see com.taobao.common.store.journal.JournalStoreMBean#getPath() */ @Override public String getPath() { return this.path; } /* * (non-Javadoc) * * @see com.taobao.common.store.journal.JournalStoreMBean#getName() */ @Override public String getName() { return this.name; } /* * (non-Javadoc) * * @see com.taobao.common.store.journal.JournalStoreMBean#getDataFileInfo() */ @Override public String getDataFileInfo() { return this.dataFile.toString(); } /* * (non-Javadoc) * * @see com.taobao.common.store.journal.JournalStoreMBean#getLogFileInfo() */ @Override public String getLogFileInfo() { return this.logFile.toString(); } /* * (non-Javadoc) * * @see com.taobao.common.store.journal.JournalStoreMBean#viewIndexMap() */ @Override public String viewIndexMap() { return this.indices.toString(); } /* * (non-Javadoc) * * @see com.taobao.common.store.Store#close() */ @Override public void close() throws IOException { this.sync(); for (final DataFile df : this.dataFiles.values()) { try { df.close(); } catch (final Exception e) { log.warn("close error:" + df, e); } } this.dataFiles.clear(); for (final LogFile lf : this.logFiles.values()) { try { lf.close(); } catch (final Exception e) { log.warn("close error:" + lf, e); } } this.logFiles.clear(); this.indices.close(); this.lastModifiedMap.clear(); this.dataFile = null; this.logFile = null; } /* * (non-Javadoc) * * @see com.taobao.common.store.journal.JournalStoreMBean#getSize() */ @Override public long getSize() throws IOException { return this.size(); } @Override public long getIntervalForCompact() { return this.intervalForCompact; } @Override public void setIntervalForCompact(final long intervalForCompact) { this.intervalForCompact = intervalForCompact; } @Override public long getIntervalForRemove() { return this.intervalForRemove; } @Override public void setIntervalForRemove(final long intervalForRemove) { this.intervalForRemove = intervalForRemove; } @Override public long getMaxFileCount() { return this.maxFileCount; } @Override public void setMaxFileCount(final long maxFileCount) { this.maxFileCount = maxFileCount; } public void sync() { this.dataFileAppender.sync(); } /** * * * 1.Remove, 2.CompactRemoveAdd * * @throws IOException */ @Override public void check() throws IOException { final Iterator<byte[]> keys = this.iterator(); BytesKey key = null; final long now = System.currentTimeMillis(); long time; log.warn("Store4j..."); while (keys.hasNext()) { key = new BytesKey(keys.next()); time = this.lastModifiedMap.get(key); if (this.intervalForRemove != -1 && now - time > this.intervalForRemove) { this.innerRemove(key.getData(), true); } else if (now - time > this.intervalForCompact) { this.reuse(key.getData(), true); } } log.warn("Store4j..."); } /** * Store4j */ class DataFileCheckThread implements Runnable { @Override public void run() { try { JournalStore.this.check(); } catch (final Exception ex) { log.warn("check error:", ex); } } } }