co.cask.cdap.data2.transaction.queue.hbase.HBaseQueueProducer.java Source code

Introduction

Here is the source code for co.cask.cdap.data2.transaction.queue.hbase.HBaseQueueProducer.java

Source

/*
 * Copyright  2014-2015 Cask Data, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */
package co.cask.cdap.data2.transaction.queue.hbase;

import co.cask.cdap.common.queue.QueueName;
import co.cask.cdap.data2.queue.ConsumerGroupConfig;
import co.cask.cdap.data2.queue.QueueEntry;
import co.cask.cdap.data2.transaction.queue.AbstractQueueProducer;
import co.cask.cdap.data2.transaction.queue.QueueEntryRow;
import co.cask.cdap.data2.transaction.queue.QueueMetrics;
import co.cask.tephra.Transaction;
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;

import java.io.Closeable;
import java.io.IOException;
import java.util.List;
import java.util.Set;

/**
 * A {@link co.cask.cdap.data2.queue.QueueProducer} that uses HBase as the storage for queue entries and consumers
 * states.
 */
public class HBaseQueueProducer extends AbstractQueueProducer implements Closeable {

    private final HBaseQueueStrategy queueStrategy;
    private final List<ConsumerGroupConfig> consumerGroupConfigs;
    private final byte[] queueRowPrefix;
    private final HTable hTable;
    private final List<byte[]> rollbackKeys;

    public HBaseQueueProducer(HTable hTable, QueueName queueName, QueueMetrics queueMetrics,
            HBaseQueueStrategy queueStrategy, Iterable<? extends ConsumerGroupConfig> consumerGroupConfigs) {
        super(queueMetrics, queueName);
        this.queueStrategy = queueStrategy;
        // Make sure only one config per consumer group
        this.consumerGroupConfigs = ImmutableList
                .copyOf(Iterables.filter(consumerGroupConfigs, new Predicate<ConsumerGroupConfig>() {
                    private final Set<Long> seenGroups = Sets.newHashSet();

                    @Override
                    public boolean apply(ConsumerGroupConfig config) {
                        return seenGroups.add(config.getGroupId());
                    }
                }));
        this.queueRowPrefix = QueueEntryRow.getQueueRowPrefix(queueName);
        this.rollbackKeys = Lists.newArrayList();
        this.hTable = hTable;
    }

    @Override
    public void startTx(Transaction tx) {
        super.startTx(tx);
        rollbackKeys.clear();
    }

    @Override
    public void close() throws IOException {
        hTable.close();
    }

    /**
     * Persist queue entries into HBase.
     */
    protected int persist(Iterable<QueueEntry> entries, Transaction transaction) throws IOException {
        int count = 0;
        List<Put> puts = Lists.newArrayList();
        int bytes = 0;

        List<byte[]> rowKeys = Lists.newArrayList();
        long writePointer = transaction.getWritePointer();
        for (QueueEntry entry : entries) {
            rowKeys.clear();
            queueStrategy.getRowKeys(consumerGroupConfigs, entry, queueRowPrefix, writePointer, count, rowKeys);
            rollbackKeys.addAll(rowKeys);

            byte[] metaData = QueueEntry.serializeHashKeys(entry.getHashKeys());
            for (byte[] rowKey : rowKeys) {
                // No need to write ts=writePointer, as the row key already contains the writePointer
                Put put = new Put(rowKey);
                put.add(QueueEntryRow.COLUMN_FAMILY, QueueEntryRow.DATA_COLUMN, entry.getData());
                put.add(QueueEntryRow.COLUMN_FAMILY, QueueEntryRow.META_COLUMN, metaData);

                puts.add(put);

                bytes += entry.getData().length;
            }
            count++;
        }
        hTable.put(puts);
        hTable.flushCommits();

        return bytes;
    }

    @Override
    protected void doRollback() throws Exception {
        // If nothing to rollback, simply return
        if (rollbackKeys.isEmpty()) {
            return;
        }

        // Delete the persisted entries
        List<Delete> deletes = Lists.newArrayList();
        for (byte[] rowKey : rollbackKeys) {
            Delete delete = new Delete(rowKey);
            deletes.add(delete);
        }
        hTable.delete(deletes);
        hTable.flushCommits();
    }
}