Java tutorial
/* * Copyright 2012 NGDATA nv * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.lilyproject.tools.recordrowvisualizer; import java.io.IOException; import java.io.OutputStream; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.NavigableMap; import freemarker.template.TemplateException; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HConnectionManager; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.util.Bytes; import org.lilyproject.bytes.impl.DataInputImpl; import org.lilyproject.cli.BaseZkCliTool; import org.lilyproject.repository.api.FieldType; import org.lilyproject.repository.api.IdGenerator; import org.lilyproject.repository.api.QName; import org.lilyproject.repository.api.RecordId; import org.lilyproject.repository.api.SchemaId; import org.lilyproject.repository.api.TypeManager; import org.lilyproject.repository.impl.EncodingUtil; import org.lilyproject.repository.impl.FieldFlags; import org.lilyproject.repository.impl.HBaseTypeManager; import org.lilyproject.repository.impl.id.IdGeneratorImpl; import org.lilyproject.repository.impl.id.SchemaIdImpl; import org.lilyproject.util.Version; import org.lilyproject.util.hbase.HBaseTableFactoryImpl; import org.lilyproject.util.hbase.LilyHBaseSchema.RecordCf; import org.lilyproject.util.hbase.LilyHBaseSchema.RecordColumn; import org.lilyproject.util.hbase.LilyHBaseSchema.Table; import org.lilyproject.util.io.Closer; import org.lilyproject.util.zookeeper.StateWatchingZooKeeper; import org.lilyproject.util.zookeeper.ZooKeeperItf; /** * Tool to visualize the HBase-storage structure of a Lily record, in the form * of an HTML page. */ public class RecordRowVisualizer extends BaseZkCliTool { protected Option recordIdOption; protected Option tableOption; protected RecordRow recordRow; protected TypeManager typeMgr; protected ZooKeeperItf zk; @Override protected String getCmdName() { return "lily-record-row-visualizer"; } @Override protected String getVersion() { return Version.readVersion("org.lilyproject", "lily-record-row-visualizer"); } @Override @SuppressWarnings("static-access") public List<Option> getOptions() { List<Option> options = super.getOptions(); recordIdOption = OptionBuilder.withArgName("record-id").hasArg() .withDescription("A Lily record ID: UUID.something or USER.something").withLongOpt("record-id") .create("r"); options.add(recordIdOption); tableOption = OptionBuilder.withArgName("table").hasArg() .withDescription("Repository table name (defaults to record)").withLongOpt("table").create("t"); options.add(tableOption); return options; } public static void main(String[] args) { new RecordRowVisualizer().start(args); } @Override public int run(CommandLine cmd) throws Exception { int result = super.run(cmd); if (result != 0) { return result; } String recordIdString = cmd.getOptionValue(recordIdOption.getOpt()); if (recordIdString == null) { System.out.println("Specify record id with -" + recordIdOption.getOpt()); return 1; } String tableName; if (cmd.hasOption(tableOption.getOpt())) { tableName = cmd.getOptionValue(tableOption.getOpt()); } else { tableName = Table.RECORD.name; } IdGenerator idGenerator = new IdGeneratorImpl(); RecordId recordId = idGenerator.fromString(recordIdString); recordRow = new RecordRow(); recordRow.recordId = recordId; // HBase record table Configuration conf = HBaseConfiguration.create(); conf.set("hbase.zookeeper.quorum", zkConnectionString); HTableInterface table = new HTable(conf, tableName); // Type manager zk = new StateWatchingZooKeeper(zkConnectionString, zkSessionTimeout); typeMgr = new HBaseTypeManager(idGenerator, conf, zk, new HBaseTableFactoryImpl(conf)); Get get = new Get(recordId.toBytes()); get.setMaxVersions(); Result row = table.get(get); NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> root = row.getMap(); readColumns(root.get(RecordCf.DATA.bytes)); byte[][] treatedColumnFamilies = { RecordCf.DATA.bytes }; for (byte[] cf : root.keySet()) { if (!isInArray(cf, treatedColumnFamilies)) { recordRow.unknownColumnFamilies.add(Bytes.toString(cf)); } } executeTemplate("recordrow2html.ftl", Collections.<String, Object>singletonMap("row", recordRow), System.out); return 0; } @Override protected void cleanup() { Closer.close(typeMgr); Closer.close(zk); HConnectionManager.deleteAllConnections(true); super.cleanup(); } private boolean isInArray(byte[] key, byte[][] data) { for (byte[] item : data) { if (Arrays.equals(item, key)) { return true; } } return false; } private void readColumns(NavigableMap<byte[], NavigableMap<Long, byte[]>> cf) throws Exception { Fields fields = recordRow.fields; for (Map.Entry<byte[], NavigableMap<Long, byte[]>> column : cf.entrySet()) { byte[] columnKey = column.getKey(); if (columnKey[0] == RecordColumn.DATA_PREFIX) { SchemaId fieldId = new SchemaIdImpl(Arrays.copyOfRange(columnKey, 1, columnKey.length)); for (Map.Entry<Long, byte[]> version : column.getValue().entrySet()) { long versionNr = version.getKey(); byte[] value = version.getValue(); FieldType fieldType = fields.registerFieldType(fieldId, typeMgr); Map<SchemaId, Object> columns = fields.values.get(versionNr); if (columns == null) { columns = new HashMap<SchemaId, Object>(); fields.values.put(versionNr, columns); } Object decodedValue; if (FieldFlags.isDeletedField(value[0])) { decodedValue = Fields.DELETED; } else { decodedValue = fieldType.getValueType() .read(new DataInputImpl(EncodingUtil.stripPrefix(value))); } columns.put(fieldId, decodedValue); } } else if (Arrays.equals(columnKey, RecordColumn.DELETED.bytes)) { setSystemField("Deleted", column.getValue(), BOOLEAN_DECODER); } else if (Arrays.equals(columnKey, RecordColumn.NON_VERSIONED_RT_ID.bytes)) { setSystemField("Non-versioned Record Type ID", column.getValue(), new RecordTypeValueDecoder(typeMgr)); } else if (Arrays.equals(columnKey, RecordColumn.NON_VERSIONED_RT_VERSION.bytes)) { setSystemField("Non-versioned Record Type Version", column.getValue(), LONG_DECODER); } else if (Arrays.equals(columnKey, RecordColumn.VERSIONED_RT_ID.bytes)) { setSystemField("Versioned Record Type ID", column.getValue(), new RecordTypeValueDecoder(typeMgr)); } else if (Arrays.equals(columnKey, RecordColumn.VERSIONED_RT_VERSION.bytes)) { setSystemField("Versioned Record Type Version", column.getValue(), LONG_DECODER); } else if (Arrays.equals(columnKey, RecordColumn.VERSIONED_MUTABLE_RT_ID.bytes)) { setSystemField("Versioned-mutable Record Type ID", column.getValue(), new RecordTypeValueDecoder(typeMgr)); } else if (Arrays.equals(columnKey, RecordColumn.VERSIONED_MUTABLE_RT_VERSION.bytes)) { setSystemField("Versioned-mutable Record Type Version", column.getValue(), LONG_DECODER); } else if (Arrays.equals(columnKey, RecordColumn.VERSION.bytes)) { setSystemField("Record Version", column.getValue(), LONG_DECODER); } else { recordRow.unknownColumns.add(Bytes.toString(columnKey)); } } } private void setSystemField(String name, NavigableMap<Long, byte[]> valuesByVersion, ValueDecoder decoder) { SystemFields systemFields = recordRow.systemFields; SystemFields.SystemField systemField = systemFields.getOrCreateSystemField(name); for (Map.Entry<Long, byte[]> entry : valuesByVersion.entrySet()) { systemField.values.put(entry.getKey(), decoder.decode(entry.getValue())); } } public static interface ValueDecoder<T> { T decode(byte[] bytes); } public static class LongValueDecoder implements ValueDecoder<Long> { @Override public Long decode(byte[] bytes) { return Bytes.toLong(bytes); } } public static class BooleanValueDecoder implements ValueDecoder<Boolean> { @Override public Boolean decode(byte[] bytes) { return Bytes.toBoolean(bytes); } } public static class StringValueDecoder implements ValueDecoder<String> { @Override public String decode(byte[] bytes) { return Bytes.toString(bytes); } } public static class Base64ValueDecoder implements ValueDecoder<String> { @Override public String decode(byte[] bytes) { if (bytes == null) { return null; } char[] result = new char[bytes.length * 2]; for (int i = 0; i < bytes.length; i++) { byte ch = bytes[i]; result[2 * i] = Character.forDigit(Math.abs(ch >> 4), 16); result[2 * i + 1] = Character.forDigit(Math.abs(ch & 0x0f), 16); } return new String(result); } } public static class RecordTypeValueDecoder implements ValueDecoder<RecordTypeInfo> { private TypeManager typeManager; public RecordTypeValueDecoder(TypeManager typeManager) { this.typeManager = typeManager; } @Override public RecordTypeInfo decode(byte[] bytes) { SchemaId id = new SchemaIdImpl(bytes); QName name; try { name = typeManager.getRecordTypeById(id, null).getName(); } catch (Exception e) { name = new QName("", "Failure retrieving record type name"); } return new RecordTypeInfo(id, name); } } private static final StringValueDecoder STRING_DECODER = new StringValueDecoder(); private static final BooleanValueDecoder BOOLEAN_DECODER = new BooleanValueDecoder(); private static final LongValueDecoder LONG_DECODER = new LongValueDecoder(); private static final Base64ValueDecoder BASE64_DECODER = new Base64ValueDecoder(); private void executeTemplate(String template, Map<String, Object> variables, OutputStream os) throws IOException, TemplateException { new TemplateRenderer().render(template, variables, os); } }