Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.lens.server.api.query; import java.io.Serializable; import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.UUID; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import org.apache.lens.api.LensConf; import org.apache.lens.api.Priority; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.driver.DriverQueryPlan; import org.apache.lens.server.api.driver.LensDriver; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.metrics.MethodMetricsContext; import org.apache.lens.server.api.metrics.MethodMetricsFactory; import org.apache.lens.server.api.query.DriverSelectorQueryContext.DriverQueryContext; import org.apache.lens.server.api.query.cost.QueryCost; import org.apache.lens.server.api.util.LensUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.session.SessionState; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @Slf4j public abstract class AbstractQueryContext implements Serializable { private static final long serialVersionUID = 1L; /** * The user query. */ @Getter protected String userQuery; /** * The replaced user query. */ @Getter @Setter protected String phase1RewrittenQuery; /** * The merged Query conf. */ @Getter @Setter protected transient Configuration conf; /** * The hive Conf. */ protected transient HiveConf hiveConf; /** * The query conf. */ @Getter protected LensConf lensConf; /** * The driver ctx */ @Getter @Setter protected transient DriverSelectorQueryContext driverContext; /** * The selected Driver query. */ protected String selectedDriverQuery; /** * The selected Driver query cost */ @Getter @Setter protected QueryCost selectedDriverQueryCost; /** * The submitted user. */ @Getter private final String submittedUser; // Logged in user. /** * The lens session identifier. */ @Getter @Setter private String lensSessionIdentifier; /** * Will be set to true when the driver queries are explicitly set * This will help avoiding rewrites in case of system restarts. */ @Getter private boolean isDriverQueryExplicitlySet = false; /** * Is olap cube query or not */ @Getter @Setter private boolean olapQuery = false; private final String database; /** Lock used to synchronize HiveConf access */ private transient Lock hiveConfLock = new ReentrantLock(); /** * The priority. */ @Getter @Setter private Priority priority; protected AbstractQueryContext(final String query, final String user, final LensConf qconf, final Configuration conf, final Collection<LensDriver> drivers, boolean mergeDriverConf) { if (conf.getBoolean(LensConfConstants.ENABLE_QUERY_METRICS, LensConfConstants.DEFAULT_ENABLE_QUERY_METRICS)) { UUID metricId = UUID.randomUUID(); conf.set(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY, metricId.toString()); log.info("Generated metric id: {} for query: {}", metricId, query); } driverContext = new DriverSelectorQueryContext(query, conf, drivers, mergeDriverConf); userQuery = query; phase1RewrittenQuery = query; this.lensConf = qconf; this.conf = conf; this.submittedUser = user; // we are setting selectedDriverQuery as user query only when the drivers size is 1 // if drivers size is more than the driver query will be set after selection over drivers if (drivers != null && drivers.size() == 1) { this.selectedDriverQuery = query; setSelectedDriver(drivers.iterator().next()); } // If this is created under an 'acquire' current db would be set if (SessionState.get() != null) { String currDb = SessionState.get().getCurrentDatabase(); database = currDb == null ? "default" : currDb; } else { database = "default"; } } // called after the object is constructed from serialized object public void initTransientState() { hiveConfLock = new ReentrantLock(); } /** * Set driver queries * * @param driverQueries Map of LensDriver to driver's query * @throws LensException */ public void setDriverQueries(Map<LensDriver, String> driverQueries) throws LensException { driverContext.setDriverQueries(driverQueries); isDriverQueryExplicitlySet = true; } /** * Estimate cost for each driver and set in context * * @throws LensException * */ public void estimateCostForDrivers() throws LensException { Map<LensDriver, DriverEstimateRunnable> estimateRunnables = getDriverEstimateRunnables(); for (LensDriver driver : estimateRunnables.keySet()) { log.info("Running estimate for driver {}", driver); estimateRunnables.get(driver).run(); } } /** * Get runnables wrapping estimate computation, which could be processed offline */ public Map<LensDriver, DriverEstimateRunnable> getDriverEstimateRunnables() throws LensException { Map<LensDriver, DriverEstimateRunnable> estimateRunnables = new HashMap<LensDriver, DriverEstimateRunnable>(); for (LensDriver driver : driverContext.getEligibleDrivers()) { estimateRunnables.put(driver, new DriverEstimateRunnable(this, driver)); } return estimateRunnables; } public Map<String, Double> getTableWeights(LensDriver driver) { return getDriverContext().getDriverRewriterPlan(driver).getTableWeights(); } public DriverQueryPlan getDriverRewriterPlan(LensDriver driver) { return getDriverContext().getDriverRewriterPlan(driver); } public String getQueue() { return getConf().get(LensConfConstants.MAPRED_JOB_QUEUE_NAME); } /** * Runnable to wrap estimate computation for a driver. Failure cause and success status * are stored as field members */ public static class DriverEstimateRunnable implements Runnable { private final AbstractQueryContext queryContext; private final LensDriver driver; @Getter private String failureCause = null; @Getter private boolean succeeded = false; @Getter private LensException cause; public DriverEstimateRunnable(AbstractQueryContext queryContext, LensDriver driver) { this.queryContext = queryContext; this.driver = driver; } @Override public void run() { MethodMetricsContext estimateGauge = MethodMetricsFactory .createMethodGauge(queryContext.getDriverConf(driver), true, "driverEstimate"); DriverQueryContext driverQueryContext = queryContext.getDriverContext().getDriverQueryContextMap() .get(driver); if (driverQueryContext.getDriverQueryRewriteError() != null) { // skip estimate return; } try { driverQueryContext.setDriverCost(driver.estimate(queryContext)); succeeded = true; } catch (final LensException e) { this.cause = e; captureExceptionInformation(driverQueryContext, e); } catch (final Exception e) { captureExceptionInformation(driverQueryContext, e); } finally { estimateGauge.markSuccess(); } } private void captureExceptionInformation(final DriverQueryContext driverQueryContext, final Exception e) { String expMsg = LensUtil.getCauseMessage(e); driverQueryContext.setDriverQueryCostEstimateError(e); failureCause = new StringBuilder("Driver :").append(driver.getFullyQualifiedName()).append(" Cause :") .append(expMsg).toString(); log.error("Setting driver cost failed for driver {} Cause: {}", driver, failureCause, e); } } /** * Wrapper method for convenience on driver context * * @return the selected driver's query */ public String getSelectedDriverQuery() { if (selectedDriverQuery != null) { return selectedDriverQuery; } else if (driverContext != null) { return driverContext.getSelectedDriverQuery(); } return null; } /** * Get driver query * * @param driver * * @return query */ public String getDriverQuery(LensDriver driver) { return driverContext.getDriverQuery(driver); } public String getFinalDriverQuery(LensDriver driver) { return driverContext.getFinalDriverQuery(driver); } /** * Get driver conf * * @param driver * * @return Configuration */ public Configuration getDriverConf(LensDriver driver) { return driverContext.getDriverConf(driver); } /** * Get query cost for the driver * * @param driver * @return QueryCostTO */ public QueryCost getDriverQueryCost(LensDriver driver) { return driverContext.getDriverQueryCost(driver); } /** * Wrapper method for convenience on driver context * * @return the selected driver's conf */ public Configuration getSelectedDriverConf() { if (driverContext != null) { return driverContext.getSelectedDriverConf(); } return null; } /** * Sets the selected driver query for persistence and also in the driver context * * @param driverQuery */ public void setSelectedDriverQuery(String driverQuery) { this.selectedDriverQuery = driverQuery; if (driverContext != null) { driverContext.setSelectedDriverQuery(driverQuery); isDriverQueryExplicitlySet = true; } } /** * Wrapper method for convenience on driver context * * @param driver Lens driver */ public void setSelectedDriver(LensDriver driver) { if (driverContext != null) { driverContext.setSelectedDriver(driver); selectedDriverQuery = driverContext.getSelectedDriverQuery(); } } /** * Wrapper method for convenience on driver context * * @return the selected driver */ public LensDriver getSelectedDriver() { if (driverContext != null) { return driverContext.getSelectedDriver(); } return null; } /** * Wrapper method for convenience on driver context * * @return the selected driver */ public DriverQueryPlan getSelectedDriverQueryPlan() throws LensException { if (driverContext != null) { return driverContext.getSelectedDriverQueryPlan(); } return null; } /** * Set exception during rewrite. * * @param driver * @param exp */ public void setDriverRewriteError(LensDriver driver, Exception exp) { driverContext.driverQueryContextMap.get(driver).setDriverQueryRewriteError(exp); } /** * Get exception during rewrite. * * @param driver * @return exp */ public Exception getDriverRewriteError(LensDriver driver) { return driverContext.driverQueryContextMap.get(driver).getDriverQueryRewriteError(); } /** * Gets HiveConf corresponding to query conf. * * Should be called judiciously, because constructing HiveConf from conf object is costly. * The field is set to null after query completion. Should not be accessed after completion. * @return */ public HiveConf getHiveConf() { hiveConfLock.lock(); try { if (hiveConf == null) { hiveConf = new HiveConf(this.conf, this.getClass()); hiveConf.setClassLoader(this.conf.getClassLoader()); } } finally { hiveConfLock.unlock(); } return hiveConf; } /** * Set final driver rewritten query for the driver. * * @param driver * @param rewrittenQuery */ public void setFinalDriverQuery(LensDriver driver, String rewrittenQuery) { driverContext.driverQueryContextMap.get(driver).setFinalDriverQuery(rewrittenQuery); } /** * Set query for a given driver * @param driver driver instance * @param query query string */ public void setDriverQuery(LensDriver driver, String query) { driverContext.setDriverQuery(driver, query); isDriverQueryExplicitlySet = true; } public void setDriverCost(LensDriver driver, QueryCost cost) { driverContext.setDriverCost(driver, cost); } /** * Get handle of the query for logging purposes * @return */ public abstract String getLogHandle(); /** * Returns database set while launching query * @return */ public String getDatabase() { return database == null ? "default" : database; } public void clearTransientStateAfterLaunch() { driverContext.clearTransientStateAfterLaunch(); } public void clearTransientStateAfterCompleted() { driverContext.clearTransientStateAfterCompleted(); hiveConf = null; } /** * Update conf. * * @param confoverlay the conf to set */ public void updateConf(Map<String, String> confoverlay) { lensConf.getProperties().putAll(confoverlay); for (Map.Entry<String, String> prop : confoverlay.entrySet()) { this.conf.set(prop.getKey(), prop.getValue()); } } }