Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hbase.client; import java.io.IOException; import java.nio.ByteBuffer; import java.util.*; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellScannable; import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.security.access.AccessControlConstants; import org.apache.hadoop.hbase.security.access.Permission; import org.apache.hadoop.hbase.security.visibility.CellVisibility; import org.apache.hadoop.hbase.security.visibility.VisibilityConstants; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ListMultimap; import com.google.common.collect.Lists; import com.google.common.io.ByteArrayDataInput; import com.google.common.io.ByteArrayDataOutput; import com.google.common.io.ByteStreams; import parquet.example.data.Group; import parquet.example.data.GroupFactory; @InterfaceAudience.Public @InterfaceStability.Evolving public abstract class Mutation extends OperationWithAttributes implements Row, CellScannable, HeapSize { private static final Log LOG = LogFactory.getLog(Mutation.class); public static final long MUTATION_OVERHEAD = ClassSize.align( // This ClassSize.OBJECT + // row + OperationWithAttributes.attributes 2 * ClassSize.REFERENCE + // Timestamp + SeqId 2 * Bytes.SIZEOF_LONG + // durability ClassSize.REFERENCE + // familyMap ClassSize.REFERENCE + // familyMap ClassSize.TREEMAP); /** * The attribute for storing the list of clusters that have consumed the change. */ private static final String CONSUMED_CLUSTER_IDS = "_cs.id"; /** * The attribute for storing TTL for the result of the mutation. */ private static final String OP_ATTRIBUTE_TTL = "_ttl"; protected byte[] row = null; protected long ts = HConstants.LATEST_TIMESTAMP; protected Durability durability = Durability.USE_DEFAULT; // A Map sorted by column family. protected NavigableMap<byte[], List<Cell>> familyMap = new TreeMap<byte[], List<Cell>>(Bytes.BYTES_COMPARATOR); //for mvcc private long seqId = 0; public long getSeqId() { return seqId; } public void setSeqId(long seqId) { this.seqId = seqId; } @Override public CellScanner cellScanner() { return CellUtil.createCellScanner(getFamilyCellMap()); } /** * Creates an empty list if one doesn't exist for the given column family * or else it returns the associated list of Cell objects. * * @param family column family * @return a list of Cell objects, returns an empty list if one doesn't exist. */ List<Cell> getCellList(byte[] family) { List<Cell> list = this.familyMap.get(family); if (list == null) { list = new ArrayList<Cell>(); } return list; } /* * Create a KeyValue with this objects row key and the Put identifier. * * @return a KeyValue with this objects row key and the Put identifier. */ KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value) { return new KeyValue(this.row, family, qualifier, ts, KeyValue.Type.Put, value); } /** * Create a KeyValue with this objects row key and the Put identifier. * * @param family * @param qualifier * @param ts * @param value * @param tags - Specify the Tags as an Array {@link KeyValue.Tag} * @return a KeyValue with this objects row key and the Put identifier. */ KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value, Tag[] tags) { KeyValue kvWithTag = new KeyValue(this.row, family, qualifier, ts, value, tags); return kvWithTag; } /* * Create a KeyValue with this objects row key and the Put identifier. * * @return a KeyValue with this objects row key and the Put identifier. */ KeyValue createPutKeyValue(byte[] family, ByteBuffer qualifier, long ts, ByteBuffer value, Tag[] tags) { return new KeyValue(this.row, 0, this.row == null ? 0 : this.row.length, family, 0, family == null ? 0 : family.length, qualifier, ts, KeyValue.Type.Put, value, tags != null ? Arrays.asList(tags) : null); } /** * Compile the column family (i.e. schema) information * into a Map. Useful for parsing and aggregation by debugging, * logging, and administration tools. * * @return Map */ @Override public Map<String, Object> getFingerprint() { Map<String, Object> map = new HashMap<String, Object>(); List<String> families = new ArrayList<String>(); // ideally, we would also include table information, but that information // is not stored in each Operation instance. map.put("families", families); for (Map.Entry<byte[], List<Cell>> entry : this.familyMap.entrySet()) { families.add(Bytes.toStringBinary(entry.getKey())); } return map; } /** * Compile the details beyond the scope of getFingerprint (row, columns, * timestamps, etc.) into a Map along with the fingerprinted information. * Useful for debugging, logging, and administration tools. * * @param maxCols a limit on the number of columns output prior to truncation * @return Map */ @Override public Map<String, Object> toMap(int maxCols) { // we start with the fingerprint map and build on top of it. Map<String, Object> map = getFingerprint(); // replace the fingerprint's simple list of families with a // map from column families to lists of qualifiers and kv details Map<String, List<Map<String, Object>>> columns = new HashMap<String, List<Map<String, Object>>>(); map.put("families", columns); map.put("row", Bytes.toStringBinary(this.row)); int colCount = 0; // iterate through all column families affected for (Map.Entry<byte[], List<Cell>> entry : this.familyMap.entrySet()) { // map from this family to details for each cell affected within the family List<Map<String, Object>> qualifierDetails = new ArrayList<Map<String, Object>>(); columns.put(Bytes.toStringBinary(entry.getKey()), qualifierDetails); colCount += entry.getValue().size(); if (maxCols <= 0) { continue; } // add details for each cell for (Cell cell : entry.getValue()) { if (--maxCols <= 0) { continue; } Map<String, Object> cellMap = cellToStringMap(cell); qualifierDetails.add(cellMap); } } map.put("totalColumns", colCount); // add the id if set if (getId() != null) { map.put("id", getId()); } // Add the TTL if set // Long.MAX_VALUE is the default, and is interpreted to mean this attribute // has not been set. if (getTTL() != Long.MAX_VALUE) { map.put("ttl", getTTL()); } return map; } private static Map<String, Object> cellToStringMap(Cell c) { Map<String, Object> stringMap = new HashMap<String, Object>(); stringMap.put("qualifier", Bytes.toStringBinary(c.getQualifierArray(), c.getQualifierOffset(), c.getQualifierLength())); stringMap.put("timestamp", c.getTimestamp()); stringMap.put("vlen", c.getValueLength()); List<Tag> tags = Tag.asList(c.getTagsArray(), c.getTagsOffset(), c.getTagsLength()); if (tags != null) { List<String> tagsString = new ArrayList<String>(); for (Tag t : tags) { tagsString.add((t.getType()) + ":" + Bytes.toStringBinary(t.getValue())); } stringMap.put("tag", tagsString); } return stringMap; } /** * @return true if edits should be applied to WAL, false if not * @deprecated Use {@link #getDurability()} instead. */ @Deprecated public boolean getWriteToWAL() { return this.durability == Durability.SKIP_WAL; } /** * Set whether this Delete should be written to the WAL or not. * Not writing the WAL means you may lose edits on server crash. * This method will reset any changes made via {@link #setDurability(Durability)} * * @param write true if edits should be written to WAL, false if not * @deprecated Use {@link #setDurability(Durability)} instead. */ @Deprecated public Mutation setWriteToWAL(boolean write) { setDurability(write ? Durability.USE_DEFAULT : Durability.SKIP_WAL); return this; } /** * Set the durability for this mutation * * @param d */ public Mutation setDurability(Durability d) { this.durability = d; return this; } /** * Get the current durability */ public Durability getDurability() { return this.durability; } /** * Method for retrieving the put's familyMap * * @return familyMap */ public NavigableMap<byte[], List<Cell>> getFamilyCellMap() { return this.familyMap; } /** * Method for setting the put's familyMap */ public Mutation setFamilyCellMap(NavigableMap<byte[], List<Cell>> map) { // TODO: Shut this down or move it up to be a Constructor. Get new object rather than change // this internal data member. this.familyMap = map; return this; } /** * Method for retrieving the put's familyMap that is deprecated and inefficient. * * @return the map * @deprecated use {@link #getFamilyCellMap()} instead. */ @Deprecated public NavigableMap<byte[], List<KeyValue>> getFamilyMap() { TreeMap<byte[], List<KeyValue>> fm = new TreeMap<byte[], List<KeyValue>>(Bytes.BYTES_COMPARATOR); for (Map.Entry<byte[], List<Cell>> e : familyMap.entrySet()) { List<KeyValue> kvl = new ArrayList<KeyValue>(e.getValue().size()); for (Cell c : e.getValue()) { kvl.add(KeyValueUtil.ensureKeyValue(c)); } fm.put(e.getKey(), kvl); } return fm; } /** * Method for setting the put's familyMap that is deprecated and inefficient. * * @deprecated use {@link #setFamilyCellMap(NavigableMap)} instead. */ @Deprecated public Mutation setFamilyMap(NavigableMap<byte[], List<KeyValue>> map) { TreeMap<byte[], List<Cell>> fm = new TreeMap<byte[], List<Cell>>(Bytes.BYTES_COMPARATOR); for (Map.Entry<byte[], List<KeyValue>> e : map.entrySet()) { fm.put(e.getKey(), Lists.<Cell>newArrayList(e.getValue())); } this.familyMap = fm; return this; } /** * Method to check if the familyMap is empty * * @return true if empty, false otherwise */ public boolean isEmpty() { return familyMap.isEmpty(); } /** * Method for retrieving the delete's row * * @return row */ @Override public byte[] getRow() { return this.row; } @Override public int compareTo(final Row d) { return Bytes.compareTo(this.getRow(), d.getRow()); } /** * Method for retrieving the timestamp * * @return timestamp */ public long getTimeStamp() { return this.ts; } /** * Marks that the clusters with the given clusterIds have consumed the mutation * * @param clusterIds of the clusters that have consumed the mutation */ public Mutation setClusterIds(List<UUID> clusterIds) { ByteArrayDataOutput out = ByteStreams.newDataOutput(); out.writeInt(clusterIds.size()); for (UUID clusterId : clusterIds) { out.writeLong(clusterId.getMostSignificantBits()); out.writeLong(clusterId.getLeastSignificantBits()); } setAttribute(CONSUMED_CLUSTER_IDS, out.toByteArray()); return this; } /** * @return the set of clusterIds that have consumed the mutation */ public List<UUID> getClusterIds() { List<UUID> clusterIds = new ArrayList<UUID>(); byte[] bytes = getAttribute(CONSUMED_CLUSTER_IDS); if (bytes != null) { ByteArrayDataInput in = ByteStreams.newDataInput(bytes); int numClusters = in.readInt(); for (int i = 0; i < numClusters; i++) { clusterIds.add(new UUID(in.readLong(), in.readLong())); } } return clusterIds; } /** * Sets the visibility expression associated with cells in this Mutation. * It is illegal to set <code>CellVisibility</code> on <code>Delete</code> mutation. * * @param expression */ public Mutation setCellVisibility(CellVisibility expression) { this.setAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY, ProtobufUtil.toCellVisibility(expression).toByteArray()); return this; } /** * @return CellVisibility associated with cells in this Mutation. * @throws DeserializationException */ public CellVisibility getCellVisibility() throws DeserializationException { byte[] cellVisibilityBytes = this.getAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY); if (cellVisibilityBytes == null) return null; return ProtobufUtil.toCellVisibility(cellVisibilityBytes); } /** * Number of KeyValues carried by this Mutation. * * @return the total number of KeyValues */ public int size() { int size = 0; for (List<Cell> cells : this.familyMap.values()) { size += cells.size(); } return size; } /** * @return the number of different families */ public int numFamilies() { return familyMap.size(); } /** * @return Calculate what Mutation adds to class heap size. */ @Override public long heapSize() { long heapsize = MUTATION_OVERHEAD; // Adding row heapsize += ClassSize.align(ClassSize.ARRAY + this.row.length); // Adding map overhead heapsize += ClassSize.align(this.familyMap.size() * ClassSize.MAP_ENTRY); for (Map.Entry<byte[], List<Cell>> entry : this.familyMap.entrySet()) { //Adding key overhead heapsize += ClassSize.align(ClassSize.ARRAY + entry.getKey().length); //This part is kinds tricky since the JVM can reuse references if you //store the same value, but have a good match with SizeOf at the moment //Adding value overhead heapsize += ClassSize.align(ClassSize.ARRAYLIST); int size = entry.getValue().size(); heapsize += ClassSize.align(ClassSize.ARRAY + size * ClassSize.REFERENCE); for (Cell cell : entry.getValue()) { heapsize += CellUtil.estimatedHeapSizeOf(cell); } } heapsize += getAttributeSize(); heapsize += extraHeapSize(); return ClassSize.align(heapsize); } /** * @return The serialized ACL for this operation, or null if none */ public byte[] getACL() { return getAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL); } /** * @param user User short name * @param perms Permissions for the user */ public Mutation setACL(String user, Permission perms) { setAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL, ProtobufUtil.toUsersAndPermissions(user, perms).toByteArray()); return this; } /** * @param perms A map of permissions for a user or users */ public Mutation setACL(Map<String, Permission> perms) { ListMultimap<String, Permission> permMap = ArrayListMultimap.create(); for (Map.Entry<String, Permission> entry : perms.entrySet()) { permMap.put(entry.getKey(), entry.getValue()); } setAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL, ProtobufUtil.toUsersAndPermissions(permMap).toByteArray()); return this; } /** * Return the TTL requested for the result of the mutation, in milliseconds. * * @return the TTL requested for the result of the mutation, in milliseconds, * or Long.MAX_VALUE if unset */ public long getTTL() { byte[] ttlBytes = getAttribute(OP_ATTRIBUTE_TTL); if (ttlBytes != null) { return Bytes.toLong(ttlBytes); } return Long.MAX_VALUE; } /** * Set the TTL desired for the result of the mutation, in milliseconds. * * @param ttl the TTL desired for the result of the mutation, in milliseconds * @return this */ public Mutation setTTL(long ttl) { setAttribute(OP_ATTRIBUTE_TTL, Bytes.toBytes(ttl)); return this; } /** * Subclasses should override this method to add the heap size of their own fields. * * @return the heap size to add (will be aligned). */ protected long extraHeapSize() { return 0L; } /** * @param row Row to check * @return <code>row</code> * @throws IllegalArgumentException Thrown if <code>row</code> is empty or null or * > {@link HConstants#MAX_ROW_LENGTH} */ static byte[] checkRow(final byte[] row) { return checkRow(row, 0, row == null ? 0 : row.length); } /** * @param row Row to check * @param offset * @param length * @return <code>row</code> * @throws IllegalArgumentException Thrown if <code>row</code> is empty or null or * > {@link HConstants#MAX_ROW_LENGTH} */ static byte[] checkRow(final byte[] row, final int offset, final int length) { if (row == null) { throw new IllegalArgumentException("Row buffer is null"); } if (length == 0) { throw new IllegalArgumentException("Row length is 0"); } if (length > HConstants.MAX_ROW_LENGTH) { throw new IllegalArgumentException("Row length " + length + " is > " + HConstants.MAX_ROW_LENGTH); } return row; } static void checkRow(ByteBuffer row) { if (row == null) { throw new IllegalArgumentException("Row buffer is null"); } if (row.remaining() == 0) { throw new IllegalArgumentException("Row length is 0"); } if (row.remaining() > HConstants.MAX_ROW_LENGTH) { throw new IllegalArgumentException( "Row length " + row.remaining() + " is > " + HConstants.MAX_ROW_LENGTH); } } /****************************************************************************************** * * for writer of pfile * @author wangxiaoyi * *******************************************************************************************/ private static final String ROW_KEY = "rowkey"; /** * transform mutation to group * assume that sfg is not null * @param gf * @return */ public Group asGroup(GroupFactory gf) { Group group = gf.newGroup().append(ROW_KEY, Bytes.toString(row)); CellScanner cellScanner = cellScanner(); try { while (cellScanner.advance()) { Cell cell = cellScanner.current(); group.append(Bytes.toString(cell.getFamily()) + ":" + Bytes.toString(cell.getQualifier()), Bytes.toString(cell.getValue())); } group.append("timestamp", System.currentTimeMillis()); } catch (IOException io) { LOG.error(io); } return group; } }