Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.solr.update; import java.io.IOException; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.util.DataInputInputStream; import org.apache.solr.common.util.FastInputStream; import org.apache.solr.common.util.FastOutputStream; import org.apache.solr.common.util.JavaBinCodec; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Log Format: List{Operation, Version, ...} * ADD, VERSION, DOC * DELETE, VERSION, ID_BYTES * DELETE_BY_QUERY, VERSION, String * * TODO: keep two files, one for [operation, version, id] and the other for the actual * document data. That way we could throw away document log files more readily * while retaining the smaller operation log files longer (and we can retrieve * the stored fields from the latest documents from the index). * * This would require keeping all source fields stored of course. * * This would also allow to not log document data for requests with commit=true * in them (since we know that if the request succeeds, all docs will be committed) * */ public class HdfsTransactionLog extends TransactionLog { public static Logger log = LoggerFactory.getLogger(HdfsTransactionLog.class); Path tlogFile; private FSDataOutputStream tlogOutStream; private FileSystem fs; HdfsTransactionLog(FileSystem fs, Path tlogFile, Collection<String> globalStrings) { this(fs, tlogFile, globalStrings, false); } HdfsTransactionLog(FileSystem fs, Path tlogFile, Collection<String> globalStrings, boolean openExisting) { super(); boolean success = false; this.fs = fs; try { if (debug) { //log.debug("New TransactionLog file=" + tlogFile + ", exists=" + tlogFile.exists() + ", size=" + tlogFile.length() + ", openExisting=" + openExisting); } this.tlogFile = tlogFile; // TODO: look into forcefully taking over any lease if (fs.exists(tlogFile) && openExisting) { tlogOutStream = fs.append(tlogFile); } else { fs.delete(tlogFile, false); tlogOutStream = fs.create(tlogFile, (short) 1); tlogOutStream.hsync(); } fos = new FastOutputStream(tlogOutStream, new byte[65536], 0); long start = tlogOutStream.getPos(); if (openExisting) { if (start > 0) { readHeader(null); // we should already be at the end // raf.seek(start); // assert channel.position() == start; fos.setWritten(start); // reflect that we aren't starting at the beginning //assert fos.size() == channel.size(); } else { addGlobalStrings(globalStrings); } } else { if (start > 0) { log.error("New transaction log already exists:" + tlogFile + " size=" + tlogOutStream.size()); } addGlobalStrings(globalStrings); } success = true; } catch (IOException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); } finally { if (!success && tlogOutStream != null) { try { tlogOutStream.close(); } catch (Exception e) { log.error("Error closing tlog file (after error opening)", e); } } } } @Override public boolean endsWithCommit() throws IOException { long size; synchronized (this) { fos.flush(); tlogOutStream.hflush(); size = fos.size(); } // the end of the file should have the end message (added during a commit) plus a 4 byte size byte[] buf = new byte[END_MESSAGE.length()]; long pos = size - END_MESSAGE.length() - 4; if (pos < 0) return false; FSDataFastInputStream dis = new FSDataFastInputStream(fs.open(tlogFile), pos); try { //ChannelFastInputStream is = new ChannelFastInputStream(channel, pos); dis.read(buf); for (int i = 0; i < buf.length; i++) { if (buf[i] != END_MESSAGE.charAt(i)) return false; } } finally { dis.close(); } return true; } // This could mess with any readers or reverse readers that are open, or anything that might try to do a log lookup. // This should only be used to roll back buffered updates, not actually applied updates. @Override public void rollback(long pos) throws IOException { synchronized (this) { assert snapshot_size == pos; fos.flush(); tlogOutStream.hflush(); // TODO: how do we rollback with hdfs?? We need HDFS-3107 //raf.setLength(pos); fos.setWritten(pos); assert fos.size() == pos; numRecords = snapshot_numRecords; } } private void readHeader(FastInputStream fis) throws IOException { // read existing header boolean closeFis = false; if (fis == null) closeFis = true; fis = fis != null ? fis : new FSDataFastInputStream(fs.open(tlogFile), 0); Map header = null; try { LogCodec codec = new LogCodec(resolver); header = (Map) codec.unmarshal(fis); fis.readInt(); // skip size } finally { if (fis != null && closeFis) { fis.close(); } } // needed to read other records synchronized (this) { globalStringList = (List<String>) header.get("strings"); globalStringMap = new HashMap<String, Integer>(globalStringList.size()); for (int i = 0; i < globalStringList.size(); i++) { globalStringMap.put(globalStringList.get(i), i + 1); } } } @Override public long writeCommit(CommitUpdateCommand cmd, int flags) { LogCodec codec = new LogCodec(resolver); synchronized (this) { try { long pos = fos.size(); // if we had flushed, this should be equal to channel.position() if (pos == 0) { writeLogHeader(codec); pos = fos.size(); } codec.init(fos); codec.writeTag(JavaBinCodec.ARR, 3); codec.writeInt(UpdateLog.COMMIT | flags); // should just take one byte codec.writeLong(cmd.getVersion()); codec.writeStr(END_MESSAGE); // ensure these bytes are (almost) last in the file endRecord(pos); fos.flush(); // flush since this will be the last record in a log fill tlogOutStream.hflush(); //assert fos.size() == channel.size(); return pos; } catch (IOException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); } } } /* This method is thread safe */ @Override public Object lookup(long pos) { // A negative position can result from a log replay (which does not re-log, but does // update the version map. This is OK since the node won't be ACTIVE when this happens. if (pos < 0) return null; try { // make sure any unflushed buffer has been flushed synchronized (this) { // TODO: optimize this by keeping track of what we have flushed up to fos.flushBuffer(); // flush to hdfs tlogOutStream.hflush(); /*** System.out.println("###flushBuffer to " + fos.size() + " raf.length()=" + raf.length() + " pos="+pos); if (fos.size() != raf.length() || pos >= fos.size() ) { throw new RuntimeException("ERROR" + "###flushBuffer to " + fos.size() + " raf.length()=" + raf.length() + " pos="+pos); } ***/ } FSDataFastInputStream dis = new FSDataFastInputStream(fs.open(tlogFile), pos); try { dis.seek(pos); LogCodec codec = new LogCodec(resolver); return codec.readVal(new FastInputStream(dis)); } finally { dis.close(); } } catch (IOException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "pos=" + pos, e); } } @Override public void finish(UpdateLog.SyncLevel syncLevel) { if (syncLevel == UpdateLog.SyncLevel.NONE) return; try { synchronized (this) { fos.flushBuffer(); // we must flush to hdfs // TODO: we probably don't need to // hsync below if we do this - I // think they are equivalent. tlogOutStream.hflush(); } if (syncLevel == UpdateLog.SyncLevel.FSYNC) { // Since fsync is outside of synchronized block, we can end up with a partial // last record on power failure (which is OK, and does not represent an error... // we just need to be aware of it when reading). //raf.getFD().sync(); tlogOutStream.hsync(); } } catch (IOException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); } } @Override protected void close() { try { if (debug) { log.debug("Closing tlog" + this); } synchronized (this) { fos.flush(); tlogOutStream.hflush(); fos.close(); tlogOutStream.close(); } if (deleteOnClose) { fs.delete(tlogFile, true); } } catch (IOException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); } } public String toString() { return "hdfs tlog{file=" + tlogFile.toString() + " refcount=" + refcount.get() + "}"; } /** Returns a reader that can be used while a log is still in use. * Currently only *one* LogReader may be outstanding, and that log may only * be used from a single thread. */ @Override public LogReader getReader(long startingPos) { return new HDFSLogReader(startingPos); } /** Returns a single threaded reverse reader */ @Override public ReverseReader getReverseReader() throws IOException { return new HDFSReverseReader(); } public class HDFSLogReader extends LogReader { FSDataFastInputStream fis; private LogCodec codec = new LogCodec(resolver); public HDFSLogReader(long startingPos) { super(); incref(); try { FSDataInputStream fdis = fs.open(tlogFile); fis = new FSDataFastInputStream(fdis, startingPos); } catch (IOException e) { throw new RuntimeException(e); } } /** Returns the next object from the log, or null if none available. * * @return The log record, or null if EOF * @throws IOException If there is a low-level I/O error. */ public Object next() throws IOException, InterruptedException { long pos = fis.position(); synchronized (HdfsTransactionLog.this) { if (trace) { log.trace("Reading log record. pos=" + pos + " currentSize=" + fos.size()); } if (pos >= fos.size()) { return null; } fos.flushBuffer(); tlogOutStream.hflush(); // we actually need a new reader fis.close(); try { FSDataInputStream fdis = fs.open(tlogFile); fis = new FSDataFastInputStream(fdis, pos); } catch (IOException e) { throw new RuntimeException(e); } } if (pos == 0) { readHeader(fis); // shouldn't currently happen - header and first record are currently written at the same time synchronized (HdfsTransactionLog.this) { if (fis.position() >= fos.size()) { return null; } pos = fis.position(); } } tlogOutStream.hflush(); Object o = codec.readVal(fis); // skip over record size int size = fis.readInt(); assert size == fis.position() - pos - 4; return o; } public void close() { try { fis.close(); } catch (IOException e) { throw new RuntimeException(e); } decref(); } @Override public String toString() { synchronized (HdfsTransactionLog.this) { return "LogReader{" + "file=" + tlogFile + ", position=" + fis.position() + ", end=" + fos.size() + "}"; } } } public class HDFSReverseReader extends ReverseReader { FSDataFastInputStream fis; private LogCodec codec = new LogCodec(resolver) { @Override public SolrInputDocument readSolrInputDocument(DataInputInputStream dis) { // Given that the SolrInputDocument is last in an add record, it's OK to just skip // reading it completely. return null; } }; int nextLength; // length of the next record (the next one closer to the start of the log file) long prevPos; // where we started reading from last time (so prevPos - nextLength == start of next record) public HDFSReverseReader() throws IOException { incref(); long sz; synchronized (HdfsTransactionLog.this) { fos.flushBuffer(); // this must be an hflush tlogOutStream.hflush(); sz = fos.size(); //assert sz == channel.size(); } fis = new FSDataFastInputStream(fs.open(tlogFile), 0); if (sz >= 4) { // readHeader(fis); // should not be needed prevPos = sz - 4; fis.seek(prevPos); nextLength = fis.readInt(); } } /** Returns the next object from the log, or null if none available. * * @return The log record, or null if EOF * @throws IOException If there is a low-level I/O error. */ public Object next() throws IOException { if (prevPos <= 0) return null; long endOfThisRecord = prevPos; int thisLength = nextLength; long recordStart = prevPos - thisLength; // back up to the beginning of the next record prevPos = recordStart - 4; // back up 4 more to read the length of the next record if (prevPos <= 0) return null; // this record is the header long bufferPos = fis.getBufferPos(); if (prevPos >= bufferPos) { // nothing to do... we're within the current buffer } else { // Position buffer so that this record is at the end. // For small records, this will cause subsequent calls to next() to be within the buffer. long seekPos = endOfThisRecord - fis.getBufferSize(); seekPos = Math.min(seekPos, prevPos); // seek to the start of the record if it's larger then the block size. seekPos = Math.max(seekPos, 0); fis.seek(seekPos); fis.peek(); // cause buffer to be filled } fis.seek(prevPos); nextLength = fis.readInt(); // this is the length of the *next* record (i.e. closer to the beginning) // TODO: optionally skip document data Object o = codec.readVal(fis); // assert fis.position() == prevPos + 4 + thisLength; // this is only true if we read all the data (and we currently skip reading SolrInputDocument return o; } /* returns the position in the log file of the last record returned by next() */ public long position() { return prevPos + 4; // skip the length } public void close() { try { fis.close(); } catch (IOException e) { throw new RuntimeException(e); } decref(); } @Override public String toString() { synchronized (HdfsTransactionLog.this) { return "LogReader{" + "file=" + tlogFile + ", position=" + fis.position() + ", end=" + fos.size() + "}"; } } } } class FSDataFastInputStream extends FastInputStream { private FSDataInputStream fis; public FSDataFastInputStream(FSDataInputStream fis, long chPosition) { // super(null, new byte[10],0,0); // a small buffer size for testing purposes super(null); this.fis = fis; super.readFromStream = chPosition; } @Override public int readWrappedStream(byte[] target, int offset, int len) throws IOException { return fis.read(readFromStream, target, offset, len); } public void seek(long position) throws IOException { if (position <= readFromStream && position >= getBufferPos()) { // seek within buffer pos = (int) (position - getBufferPos()); } else { // long currSize = ch.size(); // not needed - underlying read should handle (unless read never done) // if (position > currSize) throw new EOFException("Read past EOF: seeking to " + position + " on file of size " + currSize + " file=" + ch); readFromStream = position; end = pos = 0; } assert position() == position; } /** where is the start of the buffer relative to the whole file */ public long getBufferPos() { return readFromStream - end; } public int getBufferSize() { return buf.length; } @Override public void close() throws IOException { fis.close(); } @Override public String toString() { return "readFromStream=" + readFromStream + " pos=" + pos + " end=" + end + " bufferPos=" + getBufferPos() + " position=" + position(); } }