Java tutorial
/** * Copyright 2010 The Apache Software Foundation * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.alibaba.wasp.plan.parser.druid; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLStatement; import com.alibaba.druid.sql.ast.statement.SQLDeleteStatement; import com.alibaba.druid.sql.ast.statement.SQLInsertStatement; import com.alibaba.druid.sql.ast.statement.SQLInsertStatement.ValuesClause; import com.alibaba.druid.sql.ast.statement.SQLUpdateSetItem; import com.alibaba.druid.sql.ast.statement.SQLUpdateStatement; import com.alibaba.wasp.EntityGroupLocation; import com.alibaba.wasp.NotMatchPrimaryKeyException; import com.alibaba.wasp.ZooKeeperConnectionException; import com.alibaba.wasp.client.FConnection; import com.alibaba.wasp.client.FConnectionManager; import com.alibaba.wasp.meta.FTable; import com.alibaba.wasp.meta.Field; import com.alibaba.wasp.meta.RowBuilder; import com.alibaba.wasp.plan.DeletePlan; import com.alibaba.wasp.plan.InsertPlan; import com.alibaba.wasp.plan.UpdatePlan; import com.alibaba.wasp.plan.action.ColumnStruct; import com.alibaba.wasp.plan.action.DeleteAction; import com.alibaba.wasp.plan.action.InsertAction; import com.alibaba.wasp.plan.action.UpdateAction; import com.alibaba.wasp.plan.parser.Condition; import com.alibaba.wasp.plan.parser.ParseContext; import com.alibaba.wasp.plan.parser.UnsupportedException; import com.alibaba.wasp.plan.parser.druid.dialect.WaspSqlInsertStatement; import com.alibaba.wasp.util.ParserUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map.Entry; import java.util.Set; /** * Use Druid (https://github.com/AlibabaTech/druid) to parse the sql and * generate QueryPlan * */ public class DruidDMLParser extends DruidParser { private static final Log LOG = LogFactory.getLog(DruidDMLParser.class); private final FConnection connection; /** * @param conf * @throws com.alibaba.wasp.ZooKeeperConnectionException */ public DruidDMLParser(Configuration conf) throws ZooKeeperConnectionException { this(conf, FConnectionManager.getConnection(conf)); } /** * @param conf */ public DruidDMLParser(Configuration conf, FConnection connection) { super(); this.setConf(conf); this.connection = connection; } /** * Parse sql and generate QueryPlan SELECT,UPDATE,INSERT,DELETE */ @Override public void generatePlan(ParseContext context) throws IOException { SQLStatement stmt = context.getStmt(); MetaEventOperation metaEventOperation = new FMetaEventOperation(context.getTsr()); if (stmt instanceof SQLUpdateStatement) { // This is a Update SQL getUpdatePlan(context, (SQLUpdateStatement) stmt, metaEventOperation); } else if (stmt instanceof SQLInsertStatement) { // This is a Insert SQL getInsertPlan(context, (SQLInsertStatement) stmt, metaEventOperation); } else if (stmt instanceof SQLDeleteStatement) { // This is a Delete SQL getDeletePlan(context, (SQLDeleteStatement) stmt, metaEventOperation); } else { throw new UnsupportedException("Unsupported SQLStatement " + stmt); } } /** * Process Insert Statement and generate QueryPlan * */ private void getInsertPlan(ParseContext context, SQLInsertStatement sqlInsertStatement, MetaEventOperation metaEventOperation) throws IOException { // Parse The FROM clause String fTableName = parseFromClause(sqlInsertStatement.getTableSource()); LOG.debug("INSERT SQL TableSource " + sqlInsertStatement.getTableSource()); LOG.debug("From table " + fTableName); // check if table exists and get Table info FTable table = metaEventOperation.checkAndGetTable(fTableName, false); // Parse The columns LinkedHashSet<String> insertColumns = parseInsertColumns(sqlInsertStatement.getColumns()); LOG.debug("INSERT SQL Insert columns " + sqlInsertStatement.getColumns()); // check if table has this columns metaEventOperation.checkAndGetFields(table, insertColumns); metaEventOperation.checkRequiredFields(table, insertColumns); // Before insert a row , it should not be exists, we do not check exists // here but check it in execution. List<InsertAction> actions = new ArrayList<InsertAction>(); if (sqlInsertStatement instanceof WaspSqlInsertStatement) { // For MySQL, INSERT statements that use VALUES syntax can insert multiple // rows. see http://dev.mysql.com/doc/refman/5.5/en/insert.html List<ValuesClause> valuesList = ((WaspSqlInsertStatement) sqlInsertStatement).getValuesList(); if (valuesList.size() > 1) { throw new UnsupportedException("Insert multi value unsupported now"); } LOG.debug("INSERT SQL Insert Values List " + valuesList); for (ValuesClause values : valuesList) { actions.add(genInsertAction(metaEventOperation, table, insertColumns, values)); } } else if (sqlInsertStatement instanceof SQLInsertStatement) { // INSERT statements that use VALUES syntax insert one row ValuesClause values = sqlInsertStatement.getValues(); LOG.debug("INSERT SQL Insert Values " + values); actions.add(genInsertAction(metaEventOperation, table, insertColumns, values)); } if (context.isGenWholePlan()) { for (InsertAction insertAction : actions) { // Get entityGroupLocation according to entity group key EntityGroupLocation entityGroupLocation = this.connection.locateEntityGroup( Bytes.toBytes(table.getTableName()), insertAction.getValueOfEntityGroupKey()); insertAction.setEntityGroupLocation(entityGroupLocation); } } InsertPlan insertPlan = new InsertPlan(actions); context.setPlan(insertPlan); LOG.debug("InsertPlan " + insertPlan.toString()); } private InsertAction genInsertAction(MetaEventOperation metaEventOperation, FTable table, LinkedHashSet<String> insertColumns, ValuesClause values) throws IOException { Pair<List<Pair<String, byte[]>>, List<ColumnStruct>> pair = buildFieldsPair(metaEventOperation, table, insertColumns, values); byte[] primaryKey = RowBuilder.build().genRowkey(pair.getFirst()); return new InsertAction(table.getTableName(), pair.getFirst().get(0).getSecond(), primaryKey, pair.getSecond()); } /** * Process Delete Statement and generate QueryPlan * */ private void getDeletePlan(ParseContext context, SQLDeleteStatement sqlDeleteStatement, MetaEventOperation metaEventOperation) throws IOException { // DELETE FROM users WHERE id = 123; // Parse The FROM clause String wtableName = parseFromClause(sqlDeleteStatement.getTableSource()); LOG.debug("UPDATE SQL From clause " + sqlDeleteStatement.getTableSource()); // check if table exists and get Table info FTable table = metaEventOperation.checkAndGetTable(wtableName, false); // Parse The WHERE clause SQLExpr where = sqlDeleteStatement.getWhere(); LOG.debug("UPDATE SQL where " + where); LinkedHashMap<String, Condition> eqConditions = new LinkedHashMap<String, Condition>(); LinkedHashMap<String, Condition> ranges = new LinkedHashMap<String, Condition>(); ParserUtils.parse(where, eqConditions, ranges); if (ranges.size() > 0) { throw new UnsupportedException("RANGE is not supported!"); } // check if table has this columns metaEventOperation.checkAndGetFields(table, eqConditions.keySet()); List<Pair<String, byte[]>> primaryKeyPairs = metaEventOperation.getPrimaryKeyPairList(table, eqConditions, null); if (primaryKeyPairs == null) { throw new NotMatchPrimaryKeyException("Not match primary key."); } byte[] primayKey = RowBuilder.build().genRowkey(primaryKeyPairs); DeleteAction action = new DeleteAction(wtableName, primayKey); if (context.isGenWholePlan()) { Condition entityGroupKeyCondition = ParserUtils.getCondition(table.getEntityGroupKey().getName(), eqConditions); // Get entityGroupLocation according to entity group key EntityGroupLocation entityGroupLocation = this.connection.locateEntityGroup( Bytes.toBytes(table.getTableName()), DruidParser.convert(table.getColumn(entityGroupKeyCondition.getFieldName()), entityGroupKeyCondition.getValue())); action.setEntityGroupLocation(entityGroupLocation); } List<DeleteAction> actions = new ArrayList<DeleteAction>(); actions.add(action); DeletePlan deletePlan = new DeletePlan(actions); context.setPlan(deletePlan); LOG.debug("DeletePlan " + deletePlan.toString()); } /** * Process UPDATE Statement and generate UpdatePlan * */ private void getUpdatePlan(ParseContext context, SQLUpdateStatement sqlUpdateStatement, MetaEventOperation metaEventOperation) throws IOException { // UPDATE users SET age = 24, name = 'Mike' WHERE id = 123; // Parse The FROM clause String fTableName = parseFromClause(sqlUpdateStatement.getTableSource()); LOG.debug("UPDATE SQL From clause " + sqlUpdateStatement.getTableSource()); // check if table exists and get Table info FTable table = metaEventOperation.checkAndGetTable(fTableName, false); // Parse The WHERE clause SQLExpr where = sqlUpdateStatement.getWhere(); LOG.debug("UPDATE SQL where " + where); LinkedHashMap<String, Condition> conditions = new LinkedHashMap<String, Condition>(); LinkedHashMap<String, Condition> ranges = new LinkedHashMap<String, Condition>(); ParserUtils.parse(where, conditions, ranges); if (ranges.size() > 0) { throw new UnsupportedException("RANGE is not supported by update operation!"); } Set<String> conditionColumns = ParserUtils.getColumns(conditions); // check if table has this columns metaEventOperation.checkAndGetFields(table, conditionColumns); // check if where clause is primary keys metaEventOperation.checkIsPrimaryKey(table, conditionColumns); List<Pair<String, byte[]>> primaryKeyPairs = metaEventOperation.getPrimaryKeyPairList(table, conditions, null); if (primaryKeyPairs == null) { throw new IOException("Not match primary key."); } byte[] primayKey = RowBuilder.build().genRowkey(primaryKeyPairs); UpdateAction action = new UpdateAction(fTableName, primayKey); // Parse Update Item List<SQLUpdateSetItem> updateItems = sqlUpdateStatement.getItems(); for (SQLUpdateSetItem updateItem : updateItems) { String columnName = parseColumn(updateItem.getColumn()); // check this FTable has the column and not pk metaEventOperation.checkFieldNotInPrimaryKeys(table, columnName); Field field = table.getColumn(columnName); // Check the input is the same as DataType checkType(field, updateItem.getValue()); byte[] value = convert(field, updateItem.getValue()); String familyName = metaEventOperation.getColumnFamily(fTableName, columnName); action.addEntityColumn(fTableName, familyName, columnName, field.getType(), value); } if (context.isGenWholePlan()) { Condition entityGroupKeyCondition = ParserUtils.getCondition(table.getEntityGroupKey().getName(), conditions); // Get entityGroupLocation according to entity group key EntityGroupLocation entityGroupLocation = this.connection.locateEntityGroup( Bytes.toBytes(table.getTableName()), DruidParser.convert(table.getColumn(entityGroupKeyCondition.getFieldName()), entityGroupKeyCondition.getValue())); action.setEntityGroupLocation(entityGroupLocation); } action.setSessionId(context.getSessionId()); List<UpdateAction> actions = new ArrayList<UpdateAction>(); actions.add(action); UpdatePlan plan = new UpdatePlan(actions); context.setPlan(plan); LOG.debug("UpdatePlan " + plan); } private LinkedHashSet<String> parseInsertColumns(List<SQLExpr> columns) throws UnsupportedException { LinkedHashSet<String> columnsItem = new LinkedHashSet<String>(columns.size()); for (SQLExpr item : columns) { String columnName = parseColumn(item); LOG.debug(" SQLInsertItem " + columnName); columnsItem.add(columnName); } return columnsItem; } public Pair<List<Pair<String, byte[]>>, List<ColumnStruct>> buildFieldsPair( MetaEventOperation metaEventOperation, FTable table, LinkedHashSet<String> columns, ValuesClause values) throws IOException { // Convert a row's each field into byte[] value List<SQLExpr> exprValues = values.getValues(); if (exprValues.size() != columns.size()) { throw new IOException("Insert clause " + columns.size() + " columns " + " not match " + exprValues.size() + " values "); } Pair<String, byte[]>[] array = new Pair[table.getPrimaryKeys().size()]; // Construct all ColumnAction List<ColumnStruct> cols = new ArrayList<ColumnStruct>(columns.size()); assert (columns.size() == exprValues.size()); Iterator<String> iter = columns.iterator(); int i = 0; while (iter.hasNext()) { String columnName = iter.next(); // Get the column's info Field column = metaEventOperation.getColumnInfo(table, columnName); byte[] value = convert(column, exprValues.get(i)); Iterator<Entry<String, Field>> pkIter = table.getPrimaryKeys().entrySet().iterator(); int j = 0; while (pkIter.hasNext()) { if (pkIter.next().getKey().equalsIgnoreCase(columnName)) { array[j] = new Pair<String, byte[]>(columnName, value); break; } j++; } // Check the input is the same as DataType checkType(column, exprValues.get(i)); ColumnStruct columnAction = new ColumnStruct(table.getTableName(), column.getFamily(), columnName, column.getType(), value); cols.add(columnAction); i++; } return new Pair<List<Pair<String, byte[]>>, List<ColumnStruct>>(Arrays.asList(array), cols); } }