Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.drill.exec.store.hbase; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import java.nio.ByteBuffer; import java.nio.ByteOrder; import org.apache.drill.common.expression.CastExpression; import org.apache.drill.common.expression.ConvertExpression; import org.apache.drill.common.expression.FunctionCall; import org.apache.drill.common.expression.LogicalExpression; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.expression.ValueExpressions.BooleanExpression; import org.apache.drill.common.expression.ValueExpressions.DateExpression; import org.apache.drill.common.expression.ValueExpressions.DoubleExpression; import org.apache.drill.common.expression.ValueExpressions.FloatExpression; import org.apache.drill.common.expression.ValueExpressions.IntExpression; import org.apache.drill.common.expression.ValueExpressions.LongExpression; import org.apache.drill.common.expression.ValueExpressions.QuotedString; import org.apache.drill.common.expression.ValueExpressions.TimeExpression; import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression; import org.apache.drill.common.expression.visitors.AbstractExprVisitor; import org.apache.hadoop.hbase.util.Order; import org.apache.hadoop.hbase.util.PositionedByteRange; import org.apache.hadoop.hbase.util.SimplePositionedMutableByteRange; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.PrefixFilter; import com.google.common.base.Charsets; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpression, RuntimeException> { private byte[] value; private boolean success; private boolean isEqualityFn; private SchemaPath path; private String functionName; private boolean sortOrderAscending; // Fields for row-key prefix comparison // If the query is on row-key prefix, we cannot use a standard template to identify startRow, stopRow and filter // Hence, we use these local variables(set depending upon the encoding type in user query) private boolean isRowKeyPrefixComparison; byte[] rowKeyPrefixStartRow; byte[] rowKeyPrefixStopRow; Filter rowKeyPrefixFilter; public static boolean isCompareFunction(String functionName) { return COMPARE_FUNCTIONS_TRANSPOSE_MAP.keySet().contains(functionName); } public static CompareFunctionsProcessor process(FunctionCall call, boolean nullComparatorSupported) { String functionName = call.getName(); LogicalExpression nameArg = call.args.get(0); LogicalExpression valueArg = call.args.size() >= 2 ? call.args.get(1) : null; CompareFunctionsProcessor evaluator = new CompareFunctionsProcessor(functionName); if (valueArg != null) { // binary function if (VALUE_EXPRESSION_CLASSES.contains(nameArg.getClass())) { LogicalExpression swapArg = valueArg; valueArg = nameArg; nameArg = swapArg; evaluator.functionName = COMPARE_FUNCTIONS_TRANSPOSE_MAP.get(functionName); } evaluator.success = nameArg.accept(evaluator, valueArg); } else if (nullComparatorSupported && call.args.get(0) instanceof SchemaPath) { evaluator.success = true; evaluator.path = (SchemaPath) nameArg; } return evaluator; } public CompareFunctionsProcessor(String functionName) { this.success = false; this.functionName = functionName; this.isEqualityFn = COMPARE_FUNCTIONS_TRANSPOSE_MAP.containsKey(functionName) && COMPARE_FUNCTIONS_TRANSPOSE_MAP.get(functionName).equals(functionName); this.isRowKeyPrefixComparison = false; this.sortOrderAscending = true; } public byte[] getValue() { return value; } public boolean isSuccess() { return success; } public SchemaPath getPath() { return path; } public String getFunctionName() { return functionName; } public boolean isRowKeyPrefixComparison() { return isRowKeyPrefixComparison; } public byte[] getRowKeyPrefixStartRow() { return rowKeyPrefixStartRow; } public byte[] getRowKeyPrefixStopRow() { return rowKeyPrefixStopRow; } public Filter getRowKeyPrefixFilter() { return rowKeyPrefixFilter; } public boolean isSortOrderAscending() { return sortOrderAscending; } @Override public Boolean visitCastExpression(CastExpression e, LogicalExpression valueArg) throws RuntimeException { if (e.getInput() instanceof CastExpression || e.getInput() instanceof SchemaPath) { return e.getInput().accept(this, valueArg); } return false; } @Override public Boolean visitConvertExpression(ConvertExpression e, LogicalExpression valueArg) throws RuntimeException { if (e.getConvertFunction() == ConvertExpression.CONVERT_FROM) { String encodingType = e.getEncodingType(); int prefixLength = 0; // Handle scan pruning in the following scenario: // The row-key is a composite key and the CONVERT_FROM() function has byte_substr() as input function which is // querying for the first few bytes of the row-key(start-offset 1) // Example WHERE clause: // CONVERT_FROM(BYTE_SUBSTR(row_key, 1, 8), 'DATE_EPOCH_BE') < DATE '2015-06-17' if (e.getInput() instanceof FunctionCall) { // We can prune scan range only for big-endian encoded data if (encodingType.endsWith("_BE") == false) { return false; } FunctionCall call = (FunctionCall) e.getInput(); String functionName = call.getName(); if (!functionName.equalsIgnoreCase("byte_substr")) { return false; } LogicalExpression nameArg = call.args.get(0); LogicalExpression valueArg1 = call.args.size() >= 2 ? call.args.get(1) : null; LogicalExpression valueArg2 = call.args.size() >= 3 ? call.args.get(2) : null; if (((nameArg instanceof SchemaPath) == false) || (valueArg1 == null) || ((valueArg1 instanceof IntExpression) == false) || (valueArg2 == null) || ((valueArg2 instanceof IntExpression) == false)) { return false; } boolean isRowKey = ((SchemaPath) nameArg).getAsUnescapedPath().equals(DrillHBaseConstants.ROW_KEY); int offset = ((IntExpression) valueArg1).getInt(); if (!isRowKey || (offset != 1)) { return false; } this.path = (SchemaPath) nameArg; prefixLength = ((IntExpression) valueArg2).getInt(); this.isRowKeyPrefixComparison = true; return visitRowKeyPrefixConvertExpression(e, prefixLength, valueArg); } if (e.getInput() instanceof SchemaPath) { ByteBuf bb = null; switch (encodingType) { case "INT_BE": case "INT": case "UINT_BE": case "UINT": case "UINT4_BE": case "UINT4": if (valueArg instanceof IntExpression && (isEqualityFn || encodingType.startsWith("U"))) { bb = newByteBuf(4, encodingType.endsWith("_BE")); bb.writeInt(((IntExpression) valueArg).getInt()); } break; case "BIGINT_BE": case "BIGINT": case "UINT8_BE": case "UINT8": if (valueArg instanceof LongExpression && (isEqualityFn || encodingType.startsWith("U"))) { bb = newByteBuf(8, encodingType.endsWith("_BE")); bb.writeLong(((LongExpression) valueArg).getLong()); } break; case "FLOAT": if (valueArg instanceof FloatExpression && isEqualityFn) { bb = newByteBuf(4, true); bb.writeFloat(((FloatExpression) valueArg).getFloat()); } break; case "DOUBLE": if (valueArg instanceof DoubleExpression && isEqualityFn) { bb = newByteBuf(8, true); bb.writeDouble(((DoubleExpression) valueArg).getDouble()); } break; case "TIME_EPOCH": case "TIME_EPOCH_BE": if (valueArg instanceof TimeExpression) { bb = newByteBuf(8, encodingType.endsWith("_BE")); bb.writeLong(((TimeExpression) valueArg).getTime()); } break; case "DATE_EPOCH": case "DATE_EPOCH_BE": if (valueArg instanceof DateExpression) { bb = newByteBuf(8, encodingType.endsWith("_BE")); bb.writeLong(((DateExpression) valueArg).getDate()); } break; case "BOOLEAN_BYTE": if (valueArg instanceof BooleanExpression) { bb = newByteBuf(1, false /* does not matter */); bb.writeByte(((BooleanExpression) valueArg).getBoolean() ? 1 : 0); } break; case "DOUBLE_OB": case "DOUBLE_OBD": if (valueArg instanceof DoubleExpression) { bb = newByteBuf(9, true); PositionedByteRange br = new SimplePositionedMutableByteRange(bb.array(), 0, 9); if (encodingType.endsWith("_OBD")) { org.apache.hadoop.hbase.util.OrderedBytes.encodeFloat64(br, ((DoubleExpression) valueArg).getDouble(), Order.DESCENDING); this.sortOrderAscending = false; } else { org.apache.hadoop.hbase.util.OrderedBytes.encodeFloat64(br, ((DoubleExpression) valueArg).getDouble(), Order.ASCENDING); } } break; case "FLOAT_OB": case "FLOAT_OBD": if (valueArg instanceof FloatExpression) { bb = newByteBuf(5, true); PositionedByteRange br = new SimplePositionedMutableByteRange(bb.array(), 0, 5); if (encodingType.endsWith("_OBD")) { org.apache.hadoop.hbase.util.OrderedBytes.encodeFloat32(br, ((FloatExpression) valueArg).getFloat(), Order.DESCENDING); this.sortOrderAscending = false; } else { org.apache.hadoop.hbase.util.OrderedBytes.encodeFloat32(br, ((FloatExpression) valueArg).getFloat(), Order.ASCENDING); } } break; case "BIGINT_OB": case "BIGINT_OBD": if (valueArg instanceof LongExpression) { bb = newByteBuf(9, true); PositionedByteRange br = new SimplePositionedMutableByteRange(bb.array(), 0, 9); if (encodingType.endsWith("_OBD")) { org.apache.hadoop.hbase.util.OrderedBytes.encodeInt64(br, ((LongExpression) valueArg).getLong(), Order.DESCENDING); this.sortOrderAscending = false; } else { org.apache.hadoop.hbase.util.OrderedBytes.encodeInt64(br, ((LongExpression) valueArg).getLong(), Order.ASCENDING); } } break; case "INT_OB": case "INT_OBD": if (valueArg instanceof IntExpression) { bb = newByteBuf(5, true); PositionedByteRange br = new SimplePositionedMutableByteRange(bb.array(), 0, 5); if (encodingType.endsWith("_OBD")) { org.apache.hadoop.hbase.util.OrderedBytes.encodeInt32(br, ((IntExpression) valueArg).getInt(), Order.DESCENDING); this.sortOrderAscending = false; } else { org.apache.hadoop.hbase.util.OrderedBytes.encodeInt32(br, ((IntExpression) valueArg).getInt(), Order.ASCENDING); } } break; case "UTF8": // let visitSchemaPath() handle this. return e.getInput().accept(this, valueArg); } if (bb != null) { this.value = bb.array(); this.path = (SchemaPath) e.getInput(); return true; } } } return false; } private Boolean visitRowKeyPrefixConvertExpression(ConvertExpression e, int prefixLength, LogicalExpression valueArg) { String encodingType = e.getEncodingType(); rowKeyPrefixStartRow = HConstants.EMPTY_START_ROW; rowKeyPrefixStopRow = HConstants.EMPTY_START_ROW; rowKeyPrefixFilter = null; if ((encodingType.compareTo("UINT4_BE") == 0) || (encodingType.compareTo("UINT_BE") == 0)) { if (prefixLength != 4) { throw new RuntimeException("Invalid length(" + prefixLength + ") of row-key prefix"); } int val; if ((valueArg instanceof IntExpression) == false) { return false; } val = ((IntExpression) valueArg).getInt(); // For TIME_EPOCH_BE/BIGINT_BE encoding, the operators that we push-down are =, <>, <, <=, >, >= switch (functionName) { case "equal": rowKeyPrefixFilter = new PrefixFilter(ByteBuffer.allocate(4).putInt(val).array()); rowKeyPrefixStartRow = ByteBuffer.allocate(4).putInt(val).array(); rowKeyPrefixStopRow = ByteBuffer.allocate(4).putInt(val + 1).array(); return true; case "greater_than_or_equal_to": rowKeyPrefixStartRow = ByteBuffer.allocate(4).putInt(val).array(); return true; case "greater_than": rowKeyPrefixStartRow = ByteBuffer.allocate(4).putInt(val + 1).array(); return true; case "less_than_or_equal_to": rowKeyPrefixStopRow = ByteBuffer.allocate(4).putInt(val + 1).array(); return true; case "less_than": rowKeyPrefixStopRow = ByteBuffer.allocate(4).putInt(val).array(); return true; } return false; } if ((encodingType.compareTo("TIMESTAMP_EPOCH_BE") == 0) || (encodingType.compareTo("TIME_EPOCH_BE") == 0) || (encodingType.compareTo("UINT8_BE") == 0)) { if (prefixLength != 8) { throw new RuntimeException("Invalid length(" + prefixLength + ") of row-key prefix"); } long val; if (encodingType.compareTo("TIME_EPOCH_BE") == 0) { if ((valueArg instanceof TimeExpression) == false) { return false; } val = ((TimeExpression) valueArg).getTime(); } else if (encodingType.compareTo("UINT8_BE") == 0) { if ((valueArg instanceof LongExpression) == false) { return false; } val = ((LongExpression) valueArg).getLong(); } else if (encodingType.compareTo("TIMESTAMP_EPOCH_BE") == 0) { if ((valueArg instanceof TimeStampExpression) == false) { return false; } val = ((TimeStampExpression) valueArg).getTimeStamp(); } else { // Should not reach here. return false; } // For TIME_EPOCH_BE/BIGINT_BE encoding, the operators that we push-down are =, <>, <, <=, >, >= switch (functionName) { case "equal": rowKeyPrefixFilter = new PrefixFilter(ByteBuffer.allocate(8).putLong(val).array()); rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(val).array(); rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(val + 1).array(); return true; case "greater_than_or_equal_to": rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(val).array(); return true; case "greater_than": rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(val + 1).array(); return true; case "less_than_or_equal_to": rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(val + 1).array(); return true; case "less_than": rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(val).array(); return true; } return false; } if (encodingType.compareTo("DATE_EPOCH_BE") == 0) { if ((valueArg instanceof DateExpression) == false) { return false; } if (prefixLength != 8) { throw new RuntimeException("Invalid length(" + prefixLength + ") of row-key prefix"); } final long MILLISECONDS_IN_A_DAY = (long) 1000 * 60 * 60 * 24; long dateToSet; // For DATE encoding, the operators that we push-down are =, <>, <, <=, >, >= switch (functionName) { case "equal": long startDate = ((DateExpression) valueArg).getDate(); rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(startDate).array(); long stopDate = ((DateExpression) valueArg).getDate() + MILLISECONDS_IN_A_DAY; rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(stopDate).array(); return true; case "greater_than_or_equal_to": dateToSet = ((DateExpression) valueArg).getDate(); rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(dateToSet).array(); return true; case "greater_than": dateToSet = ((DateExpression) valueArg).getDate() + MILLISECONDS_IN_A_DAY; rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(dateToSet).array(); return true; case "less_than_or_equal_to": dateToSet = ((DateExpression) valueArg).getDate() + MILLISECONDS_IN_A_DAY; rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(dateToSet).array(); return true; case "less_than": dateToSet = ((DateExpression) valueArg).getDate(); rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(dateToSet).array(); return true; } return false; } return false; } @Override public Boolean visitUnknown(LogicalExpression e, LogicalExpression valueArg) throws RuntimeException { return false; } @Override public Boolean visitSchemaPath(SchemaPath path, LogicalExpression valueArg) throws RuntimeException { if (valueArg instanceof QuotedString) { this.value = ((QuotedString) valueArg).value.getBytes(Charsets.UTF_8); this.path = path; return true; } return false; } private static ByteBuf newByteBuf(int size, boolean bigEndian) { return Unpooled.wrappedBuffer(new byte[size]) .order(bigEndian ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN).writerIndex(0); } private static final ImmutableSet<Class<? extends LogicalExpression>> VALUE_EXPRESSION_CLASSES; static { ImmutableSet.Builder<Class<? extends LogicalExpression>> builder = ImmutableSet.builder(); VALUE_EXPRESSION_CLASSES = builder.add(BooleanExpression.class).add(DateExpression.class) .add(DoubleExpression.class).add(FloatExpression.class).add(IntExpression.class) .add(LongExpression.class).add(QuotedString.class).add(TimeExpression.class).build(); } private static final ImmutableMap<String, String> COMPARE_FUNCTIONS_TRANSPOSE_MAP; static { ImmutableMap.Builder<String, String> builder = ImmutableMap.builder(); COMPARE_FUNCTIONS_TRANSPOSE_MAP = builder // unary functions .put("isnotnull", "isnotnull").put("isNotNull", "isNotNull").put("is not null", "is not null") .put("isnull", "isnull").put("isNull", "isNull").put("is null", "is null") // binary functions .put("like", "like").put("equal", "equal").put("not_equal", "not_equal") .put("greater_than_or_equal_to", "less_than_or_equal_to").put("greater_than", "less_than") .put("less_than_or_equal_to", "greater_than_or_equal_to").put("less_than", "greater_than").build(); } }