Java tutorial
/** * Copyright 2008 The Apache Software Foundation * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hbase.regionserver.transactional; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.io.BatchOperation; import org.apache.hadoop.hbase.io.BatchUpdate; import org.apache.hadoop.hbase.regionserver.HLog; import org.apache.hadoop.hbase.regionserver.HLogEdit; import org.apache.hadoop.hbase.regionserver.HLogKey; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.util.Progressable; /** * Responsible for writing and reading (recovering) transactional information * to/from the HLog. */ class TransactionalHLogManager { private static final Log LOG = LogFactory.getLog(TransactionalHLogManager.class); private final HLog hlog; private final FileSystem fileSystem; private final HRegionInfo regionInfo; private final HBaseConfiguration conf; /** * @param region */ public TransactionalHLogManager(final TransactionalRegion region) { this.hlog = region.getLog(); this.fileSystem = region.getFilesystem(); this.regionInfo = region.getRegionInfo(); this.conf = region.getConf(); } // For Testing TransactionalHLogManager(final HLog hlog, final FileSystem fileSystem, final HRegionInfo regionInfo, final HBaseConfiguration conf) { this.hlog = hlog; this.fileSystem = fileSystem; this.regionInfo = regionInfo; this.conf = conf; } /** * @param transactionId * @throws IOException */ public void writeStartToLog(final long transactionId) throws IOException { HLogEdit logEdit; logEdit = new HLogEdit(transactionId, HLogEdit.TransactionalOperation.START); hlog.append(regionInfo, logEdit); } /** * @param transactionId * @param update * @throws IOException */ public void writeUpdateToLog(final long transactionId, final BatchUpdate update) throws IOException { long commitTime = update.getTimestamp() == HConstants.LATEST_TIMESTAMP ? System.currentTimeMillis() : update.getTimestamp(); for (BatchOperation op : update) { HLogEdit logEdit = new HLogEdit(transactionId, update.getRow(), op, commitTime); hlog.append(regionInfo, update.getRow(), logEdit); } } /** * @param transactionId * @throws IOException */ public void writeCommitToLog(final long transactionId) throws IOException { HLogEdit logEdit; logEdit = new HLogEdit(transactionId, HLogEdit.TransactionalOperation.COMMIT); hlog.append(regionInfo, logEdit); } /** * @param transactionId * @throws IOException */ public void writeAbortToLog(final long transactionId) throws IOException { HLogEdit logEdit; logEdit = new HLogEdit(transactionId, HLogEdit.TransactionalOperation.ABORT); hlog.append(regionInfo, logEdit); } /** * @param reconstructionLog * @param maxSeqID * @param reporter * @return map of batch updates * @throws UnsupportedEncodingException * @throws IOException */ public Map<Long, List<BatchUpdate>> getCommitsFromLog(final Path reconstructionLog, final long maxSeqID, final Progressable reporter) throws UnsupportedEncodingException, IOException { if (reconstructionLog == null || !fileSystem.exists(reconstructionLog)) { // Nothing to do. return null; } // Check its not empty. FileStatus[] stats = fileSystem.listStatus(reconstructionLog); if (stats == null || stats.length == 0) { LOG.warn("Passed reconstruction log " + reconstructionLog + " is zero-length"); return null; } SortedMap<Long, List<BatchUpdate>> pendingTransactionsById = new TreeMap<Long, List<BatchUpdate>>(); SortedMap<Long, List<BatchUpdate>> commitedTransactionsById = new TreeMap<Long, List<BatchUpdate>>(); Set<Long> abortedTransactions = new HashSet<Long>(); SequenceFile.Reader logReader = new SequenceFile.Reader(fileSystem, reconstructionLog, conf); try { HLogKey key = new HLogKey(); HLogEdit val = new HLogEdit(); long skippedEdits = 0; long totalEdits = 0; long startCount = 0; long writeCount = 0; long abortCount = 0; long commitCount = 0; // How many edits to apply before we send a progress report. int reportInterval = conf.getInt("hbase.hstore.report.interval.edits", 2000); while (logReader.next(key, val)) { LOG.debug("Processing edit: key: " + key.toString() + " val: " + val.toString()); if (key.getLogSeqNum() < maxSeqID) { skippedEdits++; continue; } // TODO: Change all below so we are not doing a getRow and getColumn // against a KeyValue. Each invocation creates a new instance. St.Ack. // Check this edit is for me. byte[] column = val.getKeyValue().getColumn(); Long transactionId = val.getTransactionId(); if (!val.isTransactionEntry() || HLog.isMetaColumn(column) || !Bytes.equals(key.getRegionName(), regionInfo.getRegionName())) { continue; } List<BatchUpdate> updates = pendingTransactionsById.get(transactionId); switch (val.getOperation()) { case START: if (updates != null || abortedTransactions.contains(transactionId) || commitedTransactionsById.containsKey(transactionId)) { LOG.error("Processing start for transaction: " + transactionId + ", but have already seen start message"); throw new IOException("Corrupted transaction log"); } updates = new LinkedList<BatchUpdate>(); pendingTransactionsById.put(transactionId, updates); startCount++; break; case WRITE: if (updates == null) { LOG.error("Processing edit for transaction: " + transactionId + ", but have not seen start message"); throw new IOException("Corrupted transaction log"); } BatchUpdate tranUpdate = new BatchUpdate(val.getKeyValue().getRow()); if (val.getKeyValue().getValue() != null) { tranUpdate.put(val.getKeyValue().getColumn(), val.getKeyValue().getValue()); } else { tranUpdate.delete(val.getKeyValue().getColumn()); } updates.add(tranUpdate); writeCount++; break; case ABORT: if (updates == null) { LOG.error("Processing abort for transaction: " + transactionId + ", but have not seen start message"); throw new IOException("Corrupted transaction log"); } abortedTransactions.add(transactionId); pendingTransactionsById.remove(transactionId); abortCount++; break; case COMMIT: if (updates == null) { LOG.error("Processing commit for transaction: " + transactionId + ", but have not seen start message"); throw new IOException("Corrupted transaction log"); } if (abortedTransactions.contains(transactionId)) { LOG.error("Processing commit for transaction: " + transactionId + ", but also have abort message"); throw new IOException("Corrupted transaction log"); } if (updates.size() == 0) { LOG.warn("Transaciton " + transactionId + " has no writes in log. "); } if (commitedTransactionsById.containsKey(transactionId)) { LOG.error("Processing commit for transaction: " + transactionId + ", but have already commited transaction with that id"); throw new IOException("Corrupted transaction log"); } pendingTransactionsById.remove(transactionId); commitedTransactionsById.put(transactionId, updates); commitCount++; } totalEdits++; if (reporter != null && (totalEdits % reportInterval) == 0) { reporter.progress(); } } if (LOG.isDebugEnabled()) { LOG.debug("Read " + totalEdits + " tranasctional operations (skipped " + skippedEdits + " because sequence id <= " + maxSeqID + "): " + startCount + " starts, " + writeCount + " writes, " + abortCount + " aborts, and " + commitCount + " commits."); } } finally { logReader.close(); } if (pendingTransactionsById.size() > 0) { LOG.info("Region log has " + pendingTransactionsById.size() + " unfinished transactions. Going to the transaction log to resolve"); throw new RuntimeException("Transaction log not yet implemented"); } return commitedTransactionsById; } }