Java tutorial
package com.tesora.dve.tools.analyzer; /* * #%L * Tesora Inc. * Database Virtualization Engine * %% * Copyright (C) 2011 - 2014 Tesora Inc. * %% * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License, version 3, * as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. * #L% */ import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import com.tesora.dve.server.bootstrap.BootstrapWiring; import com.tesora.dve.singleton.Singletons; import com.tesora.dve.sql.transexec.spi.TransientEngine; import com.tesora.dve.sql.transexec.spi.TransientEngineFactory; import org.apache.commons.lang.StringUtils; import com.tesora.dve.common.PECharsetUtils; import com.tesora.dve.common.PEConstants; import com.tesora.dve.db.DBNative; import com.tesora.dve.exceptions.PEException; import com.tesora.dve.sql.ParserException; import com.tesora.dve.sql.ParserException.Pass; import com.tesora.dve.sql.SchemaException; import com.tesora.dve.sql.node.expression.TableInstance; import com.tesora.dve.sql.parser.InvokeParser; import com.tesora.dve.sql.parser.ParserInvoker; import com.tesora.dve.sql.parser.ParserInvoker.LineInfo; import com.tesora.dve.sql.parser.ParserOptions; import com.tesora.dve.sql.schema.Name; import com.tesora.dve.sql.schema.PEAbstractTable; import com.tesora.dve.sql.schema.PEColumn; import com.tesora.dve.sql.schema.PEDatabase; import com.tesora.dve.sql.schema.PEForeignKey; import com.tesora.dve.sql.schema.PEForeignKeyColumn; import com.tesora.dve.sql.schema.PEKeyColumnBase; import com.tesora.dve.sql.schema.PETable; import com.tesora.dve.sql.schema.PETemplate; import com.tesora.dve.sql.schema.QualifiedName; import com.tesora.dve.sql.schema.SchemaContext; import com.tesora.dve.sql.schema.UnqualifiedName; import com.tesora.dve.sql.schema.cache.SchemaSourceFactory; import com.tesora.dve.sql.schema.modifiers.EngineTableModifier.EngineTag; import com.tesora.dve.sql.schema.validate.ValidateResult; import com.tesora.dve.sql.statement.Statement; import com.tesora.dve.sql.statement.session.UseDatabaseStatement; import com.tesora.dve.sql.statement.session.UseStatement; import com.tesora.dve.sql.statement.session.UseTenantStatement; import com.tesora.dve.sql.template.jaxb.Template; import com.tesora.dve.tools.aitemplatebuilder.CorpusStats; import com.tesora.dve.tools.analyzer.jaxb.DatabasesType.Database; import com.tesora.dve.tools.analyzer.jaxb.DbAnalyzerReport; import com.tesora.dve.tools.analyzer.jaxb.TablesType.Table; public abstract class Analyzer { static { BootstrapWiring.rewire(); } private static final int TRANSIENT_SITE_NUM = 2; private static final String TRANSIENT_SITE_USER = "root"; private static final String TRANSIENT_SITE_PASS = "password"; protected final TransientEngine tee; private final AnalyzerOptions options; protected final AnalyzerInvoker invoker; protected AnalyzerSource currentSource; protected String primaryDB; protected DbAnalyzerReport parentReport; public Analyzer(AnalyzerOptions opts) throws Throwable { if (opts == null) { throw new IllegalArgumentException(); } SchemaSourceFactory.reset(); // Build at least 2-site storage group so we guarantee we require redistribution. this.tee = buildExecutionEngine(TRANSIENT_SITE_NUM); this.options = opts; this.invoker = new AnalyzerInvoker(this); this.currentSource = null; } public void loadSchema(Template template, Database db, DBNative dbNative) throws PEException { final List<String> tableCreates = AnalyzerUtils.buildCreateTableStatements(db, dbNative); // Build view statements separately as they can depend on each other. final List<String> viewCreates = AnalyzerUtils.buildCreateViewStatements(db, dbNative); final List<String> tableNames = AnalyzerUtils.getTableNames(db); primaryDB = db.getName(); try { final List<String> dbload = new ArrayList<String>(); dbload.add("create template " + template.getName() + " xml='" + PETemplate.build(template) + "'"); dbload.add("create database " + primaryDB + " default persistent group g1 using template " + primaryDB + " strict"); dbload.add("use " + primaryDB); dbload.add("set foreign_key_checks=0"); dbload.addAll(tableCreates); dbload.add("set foreign_key_checks=1"); tee.parse(dbload.toArray(new String[dbload.size()]), true); final ParserOptions opts = ParserOptions.TEST.setResolve().setIgnoreMissingUser(); while (!viewCreates.isEmpty()) { final int before = viewCreates.size(); for (final Iterator<String> iter = viewCreates.iterator(); iter.hasNext();) { final String sql = iter.next(); final SchemaContext pc = tee.getPersistenceContext(); pc.refresh(true); try { final List<Statement> stmts = InvokeParser .parse(InvokeParser.buildInputState(sql, pc), opts, pc).getStatements(); for (final Statement s : stmts) { tee.dispatch(s); } iter.remove(); } catch (final SchemaException e) { if (e.getMessage().startsWith("No such table")) { continue; } throw new PEException("TEE: unable to parse '" + sql + "': " + e.getMessage(), e); } catch (final Exception e) { throw new PEException("TEE: unable to parse '" + sql + "': " + e.getMessage(), e); } } final int after = viewCreates.size(); if (after == before) { throw new PEException("TEE: unable to load views. Circular reference?"); } } tee.getPersistenceContext().forceMutableSource(); resolveDanglingFKs(db); if (options.isValidateFKsEnabled()) { validateFKs(primaryDB, tableNames); } } catch (final Throwable t) { throw new PEException("Unable to load schema '" + t.getMessage() + "'", t); } } private void resolveDanglingFKs(final Database db) throws PEException { final SchemaContext context = tee.getPersistenceContext(); final UnqualifiedName dbName = new UnqualifiedName(db.getName()); final PEDatabase peDb = context.findPEDatabase(dbName); if (peDb != null) { for (final Table table : db.getTables().getTable()) { if (table.isView() != Boolean.TRUE) { final UnqualifiedName tableName = new UnqualifiedName(table.getName()); final TableInstance ti = peDb.getSchema().buildInstance(context, tableName, null); if (ti != null) { final PETable peTable = ti.getAbstractTable().asTable(); final List<PEForeignKey> fks = peTable.getForeignKeys(context); for (final PEForeignKey fk : fks) { if (fk.isForward() && (fk.getTargetTable(context) == null)) { final Name targetTableName = fk.getTargetTableName(context); final TableInstance tti = peDb.getSchema().buildInstance(context, targetTableName.getUnqualified(), null); if (tti != null) { final PETable targetTable = tti.getAbstractTable().asTable(); fk.setTargetTable(context, targetTable); for (final PEKeyColumnBase keyColumn : fk.getKeyColumns()) { final PEForeignKeyColumn fkColumn = (PEForeignKeyColumn) keyColumn; final Name targetColumnName = fkColumn.getTargetColumnName(); final PEColumn targetColumn = targetTable.lookup(context, targetColumnName); if (targetColumn != null) { fkColumn.setTargetColumn(targetColumn); } else { final QualifiedName fullColumnName = new QualifiedName( targetTableName.getUnqualified(), targetColumnName.getUnqualified()); throw new PEException("TEE: target column '" + fullColumnName.getSQL() + "' not found in the schema"); } } } else { final QualifiedName fullTableName = new QualifiedName(dbName, targetTableName.getUnqualified()); throw new PEException("TEE: target table '" + fullTableName.getSQL() + "' not found in the schema"); } } } } else { final QualifiedName fullTableName = new QualifiedName(dbName, tableName); throw new PEException("TEE: table '" + fullTableName.getSQL() + "' does not exist"); } } } } else { throw new PEException("TEE: database '" + dbName.getSQL() + "' does not exist"); } } private void validateFKs(final String dbName, List<String> tableNames) { final SchemaContext sc = tee.getPersistenceContext(); final PEDatabase db = sc.findPEDatabase(new UnqualifiedName(dbName)); if (db == null) { return; } final ArrayList<ValidateResult> results = new ArrayList<ValidateResult>(); for (final String tn : tableNames) { final TableInstance ti = db.getSchema().buildInstance(sc, new UnqualifiedName(tn), null); if (ti == null) { continue; } final PEAbstractTable<?> pet = ti.getAbstractTable(); if (pet.isView()) { continue; } final List<PEForeignKey> fks = pet.asTable().getForeignKeys(sc); for (final PEForeignKey pefk : fks) { results.addAll(pefk.validate(sc, false)); } } if (results.isEmpty()) { return; } final StringBuilder errorMessage = new StringBuilder(); for (final ValidateResult vr : results) { errorMessage.append(vr.getMessage(sc)).append(PEConstants.LINE_SEPARATOR); } throw new SchemaException(Pass.PLANNER, errorMessage.toString()); } public void resolveSchemaObjects(final List<Database> databases, final CorpusStats corpusStats) { final SchemaContext context = tee.getPersistenceContext(); /* Load all tables from given static report databases. */ for (final Database staticReportDb : databases) { final String databaseName = staticReportDb.getName(); for (final Table staticReportTable : staticReportDb.getTables().getTable()) { final String tableName = staticReportTable.getName(); final int tableRowCount = staticReportTable.getRowCount(); final Long tableDataLength = staticReportTable.getDataLength(); final String engine = staticReportTable.getEngine(); corpusStats.addTable(corpusStats.new TableStats(databaseName, tableName, tableRowCount, tableDataLength, EngineTag.findEngine(engine))); } } /* Resolve additional information if available. */ for (final CorpusStats.TableStats table : corpusStats.getStatistics()) { final UnqualifiedName databaseName = new UnqualifiedName(table.getSchemaName()); final UnqualifiedName tableName = new UnqualifiedName(table.getTableName()); final PEDatabase peDb = context.findPEDatabase(databaseName); if (peDb != null) { // TODO: handle views in the analyzer final TableInstance ti = peDb.getSchema().buildInstance(context, tableName, null); final PEAbstractTable<?> tabular = ti.getAbstractTable(); if (tabular.isView() != Boolean.TRUE) { final PETable peTable = tabular.asTable(); if (peTable != null) { corpusStats.resolveTableColumns(peTable, context); corpusStats.resolveTableForeignKeys(peTable, context); } } } } } public abstract void onStatement(String sql, SourcePosition sp, Statement s) throws Throwable; public abstract void onException(String sql, SourcePosition sp, Throwable t); public abstract void onNotice(String sql, SourcePosition sp, String message); public SourcePosition convert(LineInfo li, AnalyzerSource src) { if (src != null) { return src.convert(li); } return new SourcePosition(li); } private TransientEngine buildExecutionEngine(final int numPersistentSites) throws Throwable { final TransientEngine engine = Singletons.require(TransientEngineFactory.class).create("atemp"); final List<String> persistentSiteNames = new ArrayList<String>(); final List<String> persistentDeclarations = new ArrayList<String>(); for (int i = 1; i <= numPersistentSites; ++i) { final String siteName = "site" + i; final String siteHostUrl = "jdbc:mysql://s" + i + "/db" + i; persistentDeclarations.add("create persistent site " + siteName + " url='" + siteHostUrl + "' user='" + TRANSIENT_SITE_USER + "' password='" + TRANSIENT_SITE_PASS + "'"); persistentSiteNames.add(siteName); } persistentDeclarations.add("create persistent group g1 add " + StringUtils.join(persistentSiteNames, ',')); engine.parse(persistentDeclarations.toArray(new String[] {})); return engine; } public void refresh() { tee.getPersistenceContext().refresh(true); } public AnalyzerOptions getOptions() { return options; } public ParserInvoker getInvoker() { return invoker; } public void setSource(AnalyzerSource as) { currentSource = as; } public AnalyzerSource getCurrentSource() { return currentSource; } public String getPrimaryDatabase() { return primaryDB; } @SuppressWarnings("unused") public void onFinished() throws PEException { // does nothing by default } protected static class AnalyzerInvoker extends ParserInvoker { protected Analyzer sink; private final Map<Integer, String> cdb; private Integer lastConn; public AnalyzerInvoker(Analyzer a) { super(ParserOptions.NONE); sink = a; cdb = new HashMap<Integer, String>(); lastConn = null; } @SuppressWarnings("unused") public boolean omit(LineInfo info, String line) { return false; } @Override public String parseOneLine(LineInfo info, String line) throws Throwable { final String trimmed = line.trim(); if (trimmed.equalsIgnoreCase("Connect")) { return line; } if (trimmed.equalsIgnoreCase("Quit")) { cdb.remove(info.getConnectionID()); return line; } if ((lastConn != null) && (lastConn.intValue() != info.getConnectionID())) { // we have to execute a use first final String db = cdb.get(info.getConnectionID()); if (db != null) { sink.tee.parse(new String[] { "use " + db }); } } lastConn = info.getConnectionID(); if (omit(info, line)) { return line; } final byte[] bytes = PECharsetUtils.getBytes(line, PECharsetUtils.ISO_8859_1); sink.refresh(); try { final List<Statement> stmts = InvokeParser .parse(bytes, sink.tee.getPersistenceContext(), PECharsetUtils.ISO_8859_1).getStatements(); for (final Statement s : stmts) { if (s instanceof UseStatement) { if (s instanceof UseDatabaseStatement) { final UseDatabaseStatement us = (UseDatabaseStatement) s; cdb.put(info.getConnectionID(), us.getDatabase(sink.tee.getPersistenceContext()).getName().get()); sink.tee.dispatch(s); } else if (s instanceof UseTenantStatement) { final UseTenantStatement us = (UseTenantStatement) s; cdb.put(info.getConnectionID(), us.getTenant().getExternalID()); sink.tee.dispatch(s); } } sink.onStatement(line, sink.convert(info, sink.getCurrentSource()), s); } } catch (final ParserException se) { sink.onException(line, sink.convert(info, sink.getCurrentSource()), se); } return line; } } }