Example usage for org.apache.hadoop.conf Configuration unset

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration unset.

Prototype

public synchronized void unset(String name) 

Source Link

Document

Unset a previously set property.

Usage

From source file:org.apache.lens.driver.cube.TestRewriting.java

License:Apache License

/**
 * Test cube query./*from   w w  w .  ja va2s  .  c o  m*/
 *
 * @throws ParseException    the parse exception
 * @throws SemanticException the semantic exception
 * @throws LensException     the lens exception
 */
@Test
public void testCubeQuery() throws ParseException, SemanticException, LensException {
    List<LensDriver> drivers = new ArrayList<LensDriver>();
    MockDriver driver = new MockDriver();
    LensConf lensConf = new LensConf();
    Configuration conf = new Configuration();
    driver.configure(conf);
    drivers.add(driver);

    CubeQueryRewriter mockWriter = getMockedRewriter();
    PowerMockito.stub(PowerMockito.method(RewriteUtil.class, "getCubeRewriter")).toReturn(mockWriter);
    String q1 = "select name from table";
    Assert.assertFalse(RewriteUtil.isCubeQuery(q1));
    List<RewriteUtil.CubeQueryInfo> cubeQueries = RewriteUtil.findCubePositions(q1, hconf);
    Assert.assertEquals(cubeQueries.size(), 0);
    QueryContext ctx = new QueryContext(q1, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    conf.set(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY, TestRewriting.class.getSimpleName());
    driver.configure(conf);
    String q2 = "cube select name from table";
    Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));
    MetricRegistry reg = LensMetricsRegistry.getStaticRegistry();

    Assert.assertTrue(reg.getGauges().keySet()
            .containsAll(Arrays.asList(
                    "lens.MethodMetricGauge.TestRewriting-MockDriver-RewriteUtil-rewriteQuery",
                    "lens.MethodMetricGauge.TestRewriting-MockDriver-1-RewriteUtil-rewriteQuery-toHQL")));
    conf.unset(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY);

    q2 = "insert overwrite directory '/tmp/rewrite' cube select name from table";
    Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    q2 = "insert overwrite local directory '/tmp/rewrite' cube select name from table";
    Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    q2 = "insert overwrite local directory '/tmp/example-output' cube select id,name from dim_table";
    Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query, "cube select id,name from dim_table");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    q2 = "explain cube select name from table";
    Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    q2 = "select * from (cube select name from table) a";
    Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    q2 = "insert overwrite directory '/tmp/rewrite' select * from (cube select name from table) a";
    Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    q2 = "select * from (cube select name from table)a";
    Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    q2 = "select * from  (  cube select name from table   )     a";
    Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    q2 = "select * from (      cube select name from table where"
            + " (name = 'ABC'||name = 'XYZ')&&(key=100)   )       a";
    Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
    cubeQueries = RewriteUtil.findCubePositions(RewriteUtil.getReplacedQuery(q2), hconf);
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query,
            "cube select name from" + " table where (name = 'ABC' OR name = 'XYZ') AND (key=100)");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    conf.set(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY,
            TestRewriting.class.getSimpleName() + "-multiple");
    q2 = "select * from (cube select name from table) a join (cube select" + " name2 from table2) b";
    Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 2);
    Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
    Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));
    reg = LensMetricsRegistry.getStaticRegistry();
    Assert.assertTrue(reg.getGauges().keySet()
            .containsAll(Arrays.asList(
                    "lens.MethodMetricGauge.TestRewriting-multiple-MockDriver-1-RewriteUtil-rewriteQuery-toHQL",
                    "lens.MethodMetricGauge.TestRewriting-multiple-MockDriver-2-RewriteUtil-rewriteQuery-toHQL",
                    "lens.MethodMetricGauge.TestRewriting-multiple-MockDriver-RewriteUtil-rewriteQuery")));
    conf.unset(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY);

    q2 = "select * from (cube select name from table) a full outer join"
            + " (cube select name2 from table2) b on a.name=b.name2";
    Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 2);
    Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
    Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    q2 = "select * from (cube select name from table) a join (select name2 from table2) b";
    Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    q2 = "select * from (cube select name from table union all cube select name2 from table2) u";
    Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));
    Assert.assertEquals(cubeQueries.size(), 2);
    Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
    Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");

    q2 = "insert overwrite directory '/tmp/rewrite' "
            + "select * from (cube select name from table union all cube select name2 from table2) u";
    Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));
    Assert.assertEquals(cubeQueries.size(), 2);
    Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
    Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");

    q2 = "select u.* from (select name from table    union all       cube select name2 from table2)   u";
    Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query, "cube select name2 from table2");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    q2 = "select u.* from (select name from table union all cube select name2 from table2)u";
    Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query, "cube select name2 from table2");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    q2 = "select * from (cube select name from table union all cube select name2"
            + " from table2 union all cube select name3 from table3) u";
    Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));
    Assert.assertEquals(cubeQueries.size(), 3);
    Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
    Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
    Assert.assertEquals(cubeQueries.get(2).query, "cube select name3 from table3");

    q2 = "select * from   (     cube select name from table    union all   cube"
            + " select name2 from table2   union all  cube select name3 from table3 )  u";
    Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));
    Assert.assertEquals(cubeQueries.size(), 3);
    Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
    Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
    Assert.assertEquals(cubeQueries.get(2).query, "cube select name3 from table3");

    q2 = "select * from (cube select name from table union all cube select"
            + " name2 from table2) u group by u.name";
    Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 2);
    Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
    Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    q2 = "select * from (cube select name from table union all cube select"
            + " name2 from table2)  u group by u.name";
    Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));
    Assert.assertEquals(cubeQueries.size(), 2);
    Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
    Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");

    q2 = "create table temp1 as cube select name from table";
    Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");

    q2 = "create table temp1 as select * from (cube select name from table union all cube select"
            + " name2 from table2)  u group by u.name";
    Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));
    Assert.assertEquals(cubeQueries.size(), 2);
    Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
    Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");

    q2 = "create table temp1 as cube select name from table where"
            + " time_range_in('dt', '2014-06-24-23', '2014-06-25-00')";
    Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query,
            "cube select name from table where time_range_in('dt', '2014-06-24-23', '2014-06-25-00')");

    // failing query for second driver
    MockDriver driver2 = new MockDriver();
    driver2.configure(conf);
    drivers.add(driver2);

    Assert.assertEquals(drivers.size(), 2);
    Assert.assertTrue(drivers.contains(driver) && drivers.contains(driver2));
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    Map<LensDriver, RewriteUtil.DriverRewriterRunnable> dQueries = RewriteUtil.rewriteQuery(ctx);
    Assert.assertEquals(dQueries.size(), 2);

    // Check both driver runnables are present
    Assert.assertNotNull(dQueries.get(driver), driver + " runnable not found");
    Assert.assertEquals(dQueries.get(driver).getDriver(), driver);

    Assert.assertNotNull(dQueries.get(driver2), driver2 + " runnable not found");
    Assert.assertEquals(dQueries.get(driver2).getDriver(), driver2);

    runRewrites(dQueries);

    // We have to verify that query failed for the second driver
    // First driver passes
    Iterator<LensDriver> itr = dQueries.keySet().iterator();
    itr.next();
    LensDriver failedDriver = itr.next();
    Assert.assertFalse(dQueries.get(failedDriver).isSucceeded(), failedDriver + " rewrite should have failed");
    Assert.assertNull(dQueries.get(failedDriver).getRewrittenQuery(),
            failedDriver + " rewritten query should not be set");
    // running again will fail on both drivers
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    Map<LensDriver, RewriteUtil.DriverRewriterRunnable> runnables = RewriteUtil.rewriteQuery(ctx);
    runRewrites(runnables);

    Assert.assertFalse(runnables.get(driver).isSucceeded());
    Assert.assertNotNull(runnables.get(driver).getFailureCause());
    Assert.assertNull(runnables.get(driver).getRewrittenQuery());
    Assert.assertNotNull(ctx.getDriverRewriteError(driver));

    Assert.assertFalse(runnables.get(driver2).isSucceeded());
    Assert.assertNotNull(runnables.get(driver2).getFailureCause());
    Assert.assertNull(runnables.get(driver2).getRewrittenQuery());
    Assert.assertNotNull(ctx.getDriverRewriteError(driver2));
}

---

From source file:org.apache.lens.server.rewrite.TestRewriting.java

License:Apache License

/**
 * Test cube query./*from   w  w  w.j ava 2  s  . c  o m*/
 *
 * @throws ParseException    the parse exception
 * @throws LensException     the lens exception
 * @throws HiveException
 */
@Test
public void testCubeQuery() throws ParseException, LensException, HiveException {
    SessionState.start(hconf);
    List<LensDriver> drivers = new ArrayList<LensDriver>();
    MockDriver driver = new MockDriver();
    LensConf lensConf = new LensConf();
    Configuration conf = new Configuration();
    driver.configure(conf, null, null);
    drivers.add(driver);
    List<RewriteUtil.CubeQueryInfo> cubeQueries;
    QueryContext ctx;
    String q1, q2;
    CubeQueryRewriter mockWriter = getMockedRewriter();
    CubeMetastoreClient mockClient = getMockedClient();
    PowerMockito.stub(PowerMockito.method(RewriteUtil.class, "getCubeRewriter")).toReturn(mockWriter);
    PowerMockito.stub(PowerMockito.method(RewriteUtil.class, "getClient")).toReturn(mockClient);

    q1 = "select name from table";
    cubeQueries = RewriteUtil.findCubePositions(q1, hconf);
    Assert.assertEquals(cubeQueries.size(), 0);
    ctx = new QueryContext(q1, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    conf.set(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY, TestRewriting.class.getSimpleName());
    driver.configure(conf, null, null);
    q2 = "select name from cube_table";
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query, q2);
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));
    MetricRegistry reg = LensMetricsRegistry.getStaticRegistry();

    Assert.assertTrue(reg.getGauges().keySet()
            .containsAll(Arrays.asList(
                    "lens.MethodMetricGauge.TestRewriting-" + driver.getFullyQualifiedName()
                            + "-RewriteUtil-rewriteQuery",
                    "lens.MethodMetricGauge.TestRewriting-" + driver.getFullyQualifiedName()
                            + "-1-RewriteUtil-rewriteQuery-toHQL")));
    conf.unset(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY);

    assertIsCubeQuery("select name from cube_table", lensConf, conf, drivers);
    assertIsCubeQuery("select cube_table.name from cube_table", lensConf, conf, drivers);
    assertIsCubeQuery("select cube_dim.name, cube_dim2.name from cube_dim "
            + "join cube_dim2 on cube_dim.d2id=cube_dim2.id", lensConf, conf, drivers);
    assertIsCubeQuery("select distinct cube_dim.name, cube_dim2.name from cube_dim full outer "
            + "join cube_dim2 on cube_dim.d2id=cube_dim2.id", lensConf, conf, drivers);
    assertIsCubeQuery("select cube_dim.name, cube_dim2.name from cube_dim left outer "
            + "join cube_dim2 on cube_dim.d2id=cube_dim2.id", lensConf, conf, drivers);
    assertIsCubeQuery("select cube_dim.name, cube_dim2.name from cube_dim inner "
            + "join cube_dim2 on cube_dim.d2id=cube_dim2.id", lensConf, conf, drivers);

    q2 = "select * from (select name from cube_table) table1";
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    q2 = "insert overwrite directory 'target/rewrite' select name from cube_table";
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    q2 = "insert overwrite local directory 'target/example-output' select id,name from cube_dim_table";
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query, "select id,name from cube_dim_table");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    q2 = "explain select name from cube_table";
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    q2 = "select * from (select name from cube_table) a";
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    q2 = "insert overwrite directory 'target/rewrite' select * from (select name from cube_table) a";
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    q2 = "select * from (select name from cube_table)a";
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    q2 = "select * from  (  select name from cube_table   )     a";
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    q2 = "select * from  (  select name from cube_table   )     a";
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    q2 = "select * from (      select name from cube_table where"
            + " (name = 'ABC'||name = 'XYZ')&&(key=100)   )       a";
    cubeQueries = RewriteUtil.findCubePositions(RewriteUtil.getReplacedQuery(q2), hconf);
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query,
            "select name from" + " cube_table where (name = 'ABC' OR name = 'XYZ') AND (key=100)");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    conf.set(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY,
            TestRewriting.class.getSimpleName() + "-multiple");
    q2 = "select * from (select name from cube_table) a join (select" + " name2 from cube_table2) b";
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 2);
    Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table");
    Assert.assertEquals(cubeQueries.get(1).query, "select name2 from cube_table2");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));
    reg = LensMetricsRegistry.getStaticRegistry();
    Assert.assertTrue(reg.getGauges().keySet()
            .containsAll(Arrays.asList(
                    "lens.MethodMetricGauge.TestRewriting-" + driver.getFullyQualifiedName()
                            + "-1-RewriteUtil-rewriteQuery-toHQL",
                    "lens.MethodMetricGauge.TestRewriting-multiple-" + driver.getFullyQualifiedName()
                            + "-2-RewriteUtil-rewriteQuery-toHQL",
                    "lens.MethodMetricGauge.TestRewriting-multiple-" + driver.getFullyQualifiedName()
                            + "-RewriteUtil-rewriteQuery")));
    conf.unset(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY);

    q2 = "select * from (select name from cube_table) a full outer join"
            + " (select name2 from cube_table2) b on a.name=b.name2";
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 2);
    Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table");
    Assert.assertEquals(cubeQueries.get(1).query, "select name2 from cube_table2");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    q2 = "select * from (select name from cube_table) a join (select name2 from table2) b";
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    q2 = "select * from (select name from cube_table) a join (select name2 from cube_table2) b";
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 2);
    Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table");
    Assert.assertEquals(cubeQueries.get(1).query, "select name2 from cube_table2");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    q2 = "select * from (select name from cube_table union all select name2 from cube_table2) u";
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));
    Assert.assertEquals(cubeQueries.size(), 2);
    Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table");
    Assert.assertEquals(cubeQueries.get(1).query, "select name2 from cube_table2");

    q2 = "select * from (select name from cube_table union all select distinct name2 from cube_table2) u";
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));
    Assert.assertEquals(cubeQueries.size(), 2);
    Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table");
    Assert.assertEquals(cubeQueries.get(1).query, "select distinct name2 from cube_table2");

    q2 = "select * from (select distinct name from cube_table union all select name2 from cube_table2) u";
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));
    Assert.assertEquals(cubeQueries.size(), 2);
    Assert.assertEquals(cubeQueries.get(0).query, "select distinct name from cube_table");
    Assert.assertEquals(cubeQueries.get(1).query, "select name2 from cube_table2");

    q2 = "insert overwrite directory 'target/rewrite' "
            + "select * from (select name from cube_table union all select name2 from cube_table2) u";
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));
    Assert.assertEquals(cubeQueries.size(), 2);
    Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table");
    Assert.assertEquals(cubeQueries.get(1).query, "select name2 from cube_table2");

    q2 = "select u.* from (select name from table    union all       select name2 from cube_table2)   u";
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query, "select name2 from cube_table2");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    q2 = "select u.* from (select name from table union all select name2 from cube_table2)u";
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query, "select name2 from cube_table2");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    q2 = "select * from (select name from cube_table union all select name2"
            + " from cube_table2 union all select name3 from cube_table3) u";
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));
    Assert.assertEquals(cubeQueries.size(), 3);
    Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table");
    Assert.assertEquals(cubeQueries.get(1).query, "select name2 from cube_table2");
    Assert.assertEquals(cubeQueries.get(2).query, "select name3 from cube_table3");

    q2 = "select * from (select name from cube_table union all select"
            + " name2 from cube_table2) u group by u.name";
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    Assert.assertEquals(cubeQueries.size(), 2);
    Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table");
    Assert.assertEquals(cubeQueries.get(1).query, "select name2 from cube_table2");
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));

    q2 = "select * from (select name from cube_table union all select"
            + " name2 from cube_table2)  u group by u.name";
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));
    Assert.assertEquals(cubeQueries.size(), 2);
    Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table");
    Assert.assertEquals(cubeQueries.get(1).query, "select name2 from cube_table2");

    q2 = "create table temp1 as select name from cube_table";
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table");

    q2 = "create table temp1 as select * from (select name from cube_table union all select"
            + " name2 from cube_table2)  u group by u.name";
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));
    Assert.assertEquals(cubeQueries.size(), 2);
    Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table");
    Assert.assertEquals(cubeQueries.get(1).query, "select name2 from cube_table2");

    q2 = "create table temp1 as select name from cube_table where"
            + " time_range_in('dt', '2014-06-24-23', '2014-06-25-00')";
    cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    runRewrites(RewriteUtil.rewriteQuery(ctx));
    Assert.assertEquals(cubeQueries.size(), 1);
    Assert.assertEquals(cubeQueries.get(0).query,
            "select name from cube_table where time_range_in('dt', '2014-06-24-23', '2014-06-25-00')");

    q2 = "SELECT case when a='b' then 'c' else 'd' end some_field from cube_table";
    cubeQueries = RewriteUtil.findCubePositions(q2 + " union all " + q2, hconf);

    Assert.assertEquals(cubeQueries.size(), 2);
    Assert.assertEquals(cubeQueries.get(0).query, q2);
    Assert.assertEquals(cubeQueries.get(1).query, q2);

    // failing query for second driver
    MockDriver driver2 = new MockDriver();
    driver2.configure(conf, null, null);
    drivers.add(driver2);

    Assert.assertEquals(drivers.size(), 2);
    Assert.assertTrue(drivers.contains(driver) && drivers.contains(driver2));
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    Map<LensDriver, RewriteUtil.DriverRewriterRunnable> dQueries = RewriteUtil.rewriteQuery(ctx);
    Assert.assertEquals(dQueries.size(), 2);

    // Check both driver runnables are present
    Assert.assertNotNull(dQueries.get(driver), driver + " runnable not found");
    Assert.assertEquals(dQueries.get(driver).getDriver(), driver);

    Assert.assertNotNull(dQueries.get(driver2), driver2 + " runnable not found");
    Assert.assertEquals(dQueries.get(driver2).getDriver(), driver2);

    runRewrites(dQueries);

    // We have to verify that query failed for the second driver
    // First driver passes
    Iterator<LensDriver> itr = dQueries.keySet().iterator();
    itr.next();
    LensDriver failedDriver = itr.next();
    Assert.assertFalse(dQueries.get(failedDriver).isSucceeded(), failedDriver + " rewrite should have failed");
    Assert.assertNull(dQueries.get(failedDriver).getRewrittenQuery(),
            failedDriver + " rewritten query should not be set");
    // running again will fail on both drivers
    ctx = new QueryContext(q2, null, lensConf, conf, drivers);
    Map<LensDriver, RewriteUtil.DriverRewriterRunnable> runnables = RewriteUtil.rewriteQuery(ctx);
    runRewrites(runnables);

    Assert.assertFalse(runnables.get(driver).isSucceeded());
    Assert.assertNotNull(runnables.get(driver).getFailureCause());
    Assert.assertNull(runnables.get(driver).getRewrittenQuery());
    Assert.assertNotNull(ctx.getDriverRewriteError(driver));

    Assert.assertFalse(runnables.get(driver2).isSucceeded());
    Assert.assertNotNull(runnables.get(driver2).getFailureCause());
    Assert.assertNull(runnables.get(driver2).getRewrittenQuery());
    Assert.assertNotNull(ctx.getDriverRewriteError(driver2));
}

---

From source file:org.apache.oozie.command.coord.CoordActionStartXCommand.java

License:Apache License

/**
 * Create config to pass to WF Engine 1. Get createdConf from coord_actions table 2. Get actionXml from
 * coord_actions table. Extract all 'property' tags and merge createdConf (overwrite duplicate keys). 3. Extract
 * 'app-path' from actionXML. Create a new property called 'oozie.wf.application.path' and merge with createdConf
 * (overwrite duplicate keys) 4. Read contents of config-default.xml in workflow directory. 5. Merge createdConf
 * with config-default.xml (overwrite duplicate keys). 6. Results is runConf which is saved in coord_actions table.
 * Merge Action createdConf with actionXml to create new runConf with replaced variables
 *
 * @param action CoordinatorActionBean/*from  w w w  .ja  v a  2  s.co m*/
 * @return Configuration
 * @throws CommandException
 */
private Configuration mergeConfig(CoordinatorActionBean action) throws CommandException {
    String createdConf = action.getCreatedConf();
    String actionXml = action.getActionXml();
    Element workflowProperties = null;
    try {
        workflowProperties = XmlUtils.parseXml(actionXml);
    } catch (JDOMException e1) {
        log.warn("Configuration parse error in:" + actionXml);
        throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1);
    }
    // generate the 'runConf' for this action
    // Step 1: runConf = createdConf
    Configuration runConf = null;
    try {
        runConf = new XConfiguration(new StringReader(createdConf));

    } catch (IOException e1) {
        log.warn("Configuration parse error in:" + createdConf);
        throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1);
    }
    // Step 2: Merge local properties into runConf
    // extract 'property' tags under 'configuration' block in the
    // coordinator.xml (saved in actionxml column)
    // convert Element to XConfiguration
    Element configElement = workflowProperties.getChild("action", workflowProperties.getNamespace())
            .getChild("workflow", workflowProperties.getNamespace())
            .getChild("configuration", workflowProperties.getNamespace());
    if (configElement != null) {
        String strConfig = XmlUtils.prettyPrint(configElement).toString();
        Configuration localConf;
        try {
            localConf = new XConfiguration(new StringReader(strConfig));
        } catch (IOException e1) {
            log.warn("Configuration parse error in:" + strConfig);
            throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1);
        }

        // copy configuration properties in coordinator.xml to the runConf
        XConfiguration.copy(localConf, runConf);
    }

    // Step 3: Extract value of 'app-path' in actionxml, and save it as a
    // new property called 'oozie.wf.application.path'
    // WF Engine requires the path to the workflow.xml to be saved under
    // this property name
    String appPath = workflowProperties.getChild("action", workflowProperties.getNamespace())
            .getChild("workflow", workflowProperties.getNamespace())
            .getChild("app-path", workflowProperties.getNamespace()).getValue();

    // Copying application path in runconf.
    runConf.set("oozie.wf.application.path", appPath);

    // Step 4: Extract the runconf and copy the rerun config to runconf.
    if (runConf.get(CoordRerunXCommand.RERUN_CONF) != null) {
        Configuration rerunConf = null;
        try {
            rerunConf = new XConfiguration(new StringReader(runConf.get(CoordRerunXCommand.RERUN_CONF)));
            XConfiguration.copy(rerunConf, runConf);
        } catch (IOException e) {
            log.warn("Configuration parse error in:" + rerunConf);
            throw new CommandException(ErrorCode.E1005, e.getMessage(), e);
        }
        runConf.unset(CoordRerunXCommand.RERUN_CONF);
    }
    return runConf;
}

---

From source file:org.apache.oozie.servlet.BaseJobServlet.java

License:Apache License

/**
 * Perform various job related actions - start, suspend, resume, kill, etc.
 *///from   w w w.  j a va2s  . co m
@Override
protected void doPut(HttpServletRequest request, HttpServletResponse response)
        throws ServletException, IOException {
    String jobId = getResourceName(request);
    request.setAttribute(AUDIT_PARAM, jobId);
    request.setAttribute(AUDIT_OPERATION, request.getParameter(RestConstants.ACTION_PARAM));
    try {
        AuthorizationService auth = Services.get().get(AuthorizationService.class);
        auth.authorizeForJob(getUser(request), jobId, true);
    } catch (AuthorizationException ex) {
        throw new XServletException(HttpServletResponse.SC_UNAUTHORIZED, ex);
    }

    String action = request.getParameter(RestConstants.ACTION_PARAM);
    if (action.equals(RestConstants.JOB_ACTION_START)) {
        stopCron();
        startJob(request, response);
        startCron();
        response.setStatus(HttpServletResponse.SC_OK);
    } else if (action.equals(RestConstants.JOB_ACTION_RESUME)) {
        stopCron();
        resumeJob(request, response);
        startCron();
        response.setStatus(HttpServletResponse.SC_OK);
    } else if (action.equals(RestConstants.JOB_ACTION_SUSPEND)) {
        stopCron();
        suspendJob(request, response);
        startCron();
        response.setStatus(HttpServletResponse.SC_OK);
    } else if (action.equals(RestConstants.JOB_ACTION_KILL)) {
        stopCron();
        JSONObject json = killJob(request, response);
        startCron();
        if (json != null) {
            sendJsonResponse(response, HttpServletResponse.SC_OK, json);
        } else {
            response.setStatus(HttpServletResponse.SC_OK);
        }
    } else if (action.equals(RestConstants.JOB_ACTION_CHANGE)) {
        stopCron();
        changeJob(request, response);
        startCron();
        response.setStatus(HttpServletResponse.SC_OK);
    } else if (action.equals(RestConstants.JOB_ACTION_IGNORE)) {
        stopCron();
        JSONObject json = ignoreJob(request, response);
        startCron();
        if (json != null) {
            sendJsonResponse(response, HttpServletResponse.SC_OK, json);
        } else {
            response.setStatus(HttpServletResponse.SC_OK);
        }
    } else if (action.equals(RestConstants.JOB_ACTION_RERUN)) {
        validateContentType(request, RestConstants.XML_CONTENT_TYPE);
        Configuration conf = new XConfiguration(request.getInputStream());
        stopCron();
        String requestUser = getUser(request);
        if (!requestUser.equals(UNDEF)) {
            conf.set(OozieClient.USER_NAME, requestUser);
        }
        if (conf.get(OozieClient.APP_PATH) != null) {
            BaseJobServlet.checkAuthorizationForApp(conf);
            JobUtils.normalizeAppPath(conf.get(OozieClient.USER_NAME), conf.get(OozieClient.GROUP_NAME), conf);
        }
        reRunJob(request, response, conf);
        startCron();
        response.setStatus(HttpServletResponse.SC_OK);
    } else if (action.equals(RestConstants.JOB_COORD_ACTION_RERUN)) {
        validateContentType(request, RestConstants.XML_CONTENT_TYPE);
        stopCron();
        JSONObject json = reRunJob(request, response, null);
        startCron();
        if (json != null) {
            sendJsonResponse(response, HttpServletResponse.SC_OK, json);
        } else {
            response.setStatus(HttpServletResponse.SC_OK);
        }
    } else if (action.equals(RestConstants.JOB_BUNDLE_ACTION_RERUN)) {
        validateContentType(request, RestConstants.XML_CONTENT_TYPE);
        stopCron();
        JSONObject json = reRunJob(request, response, null);
        startCron();
        if (json != null) {
            sendJsonResponse(response, HttpServletResponse.SC_OK, json);
        } else {
            response.setStatus(HttpServletResponse.SC_OK);
        }
    } else if (action.equals(RestConstants.JOB_COORD_UPDATE)) {
        validateContentType(request, RestConstants.XML_CONTENT_TYPE);
        Configuration conf = new XConfiguration(request.getInputStream());
        stopCron();
        String requestUser = getUser(request);
        if (!requestUser.equals(UNDEF)) {
            conf.set(OozieClient.USER_NAME, requestUser);
        }
        if (conf.get(OozieClient.COORDINATOR_APP_PATH) != null) {
            //If coord is submitted from bundle, user may want to update individual coord job with bundle properties
            //If COORDINATOR_APP_PATH is set, we should check only COORDINATOR_APP_PATH path permission
            String bundlePath = conf.get(OozieClient.BUNDLE_APP_PATH);
            if (bundlePath != null) {
                conf.unset(OozieClient.BUNDLE_APP_PATH);
            }
            BaseJobServlet.checkAuthorizationForApp(conf);
            JobUtils.normalizeAppPath(conf.get(OozieClient.USER_NAME), conf.get(OozieClient.GROUP_NAME), conf);
            if (bundlePath != null) {
                conf.set(OozieClient.BUNDLE_APP_PATH, bundlePath);
            }
        }
        JSONObject json = updateJob(request, response, conf);
        startCron();
        sendJsonResponse(response, HttpServletResponse.SC_OK, json);
    } else if (action.equals(RestConstants.SLA_ENABLE_ALERT)) {
        validateContentType(request, RestConstants.XML_CONTENT_TYPE);
        stopCron();
        slaEnableAlert(request, response);
        startCron();
        response.setStatus(HttpServletResponse.SC_OK);
    } else if (action.equals(RestConstants.SLA_DISABLE_ALERT)) {
        validateContentType(request, RestConstants.XML_CONTENT_TYPE);
        stopCron();
        slaDisableAlert(request, response);
        startCron();
        response.setStatus(HttpServletResponse.SC_OK);
    } else if (action.equals(RestConstants.SLA_CHANGE)) {
        validateContentType(request, RestConstants.XML_CONTENT_TYPE);
        stopCron();
        slaChange(request, response);
        startCron();
        response.setStatus(HttpServletResponse.SC_OK);
    } else {
        throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0303,
                RestConstants.ACTION_PARAM, action);
    }
}

---

From source file:org.apache.phoenix.transaction.TephraTransactionContext.java

License:Apache License

@Override
public void setTxnConfigs(Configuration config, String tmpFolder, int defaultTxnTimeoutSeconds)
        throws IOException {
    config.setBoolean(TxConstants.Manager.CFG_DO_PERSIST, false);
    config.set(TxConstants.Service.CFG_DATA_TX_CLIENT_RETRY_STRATEGY, "n-times");
    config.setInt(TxConstants.Service.CFG_DATA_TX_CLIENT_ATTEMPTS, 1);
    config.setInt(TxConstants.Service.CFG_DATA_TX_BIND_PORT, Networks.getRandomPort());
    config.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, tmpFolder);
    config.setInt(TxConstants.Manager.CFG_TX_TIMEOUT, defaultTxnTimeoutSeconds);
    config.unset(TxConstants.Manager.CFG_TX_HDFS_USER);
    config.setLong(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL, 5L);
}

---

From source file:org.apache.pig.backend.hadoop.executionengine.shims.HadoopShims.java

License:Apache License

public static void unsetConf(Configuration conf, String key) {
    conf.unset(key);
}

---

From source file:org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper.java

License:Apache License

/**
 * Convert MR settings to Tez settings and set on conf.
 *
 * @param conf  Configuration on which MR equivalent Tez settings should be set
 * @param mrConf Configuration that contains MR settings
 *//*from  w  w  w.ja  v  a  2 s. co m*/
private static void convertMRToTezRuntimeConf(Configuration conf, Configuration mrConf) {
    for (Entry<String, String> dep : DeprecatedKeys.getMRToTezRuntimeParamMap().entrySet()) {
        if (mrConf.get(dep.getKey()) != null) {
            conf.unset(dep.getKey());
            LOG.info("Setting " + dep.getValue() + " to " + mrConf.get(dep.getKey()) + " from MR setting "
                    + dep.getKey());
            conf.setIfUnset(dep.getValue(), mrConf.get(dep.getKey()));
        }
    }
}

---

From source file:org.apache.tez.common.TestTezCommonUtils.java

License:Apache License

@Test(timeout = 5000)
public void testTezBaseStagingPath() throws Exception {
    Configuration localConf = new Configuration();
    // Check if default works with localFS
    localConf.unset(TezConfiguration.TEZ_AM_STAGING_DIR);
    localConf.set("fs.defaultFS", "file:///");
    Path stageDir = TezCommonUtils.getTezBaseStagingPath(localConf);
    Assert.assertEquals(stageDir.toString(), "file:" + TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT);

    // check if user set something, indeed works
    conf.set(TezConfiguration.TEZ_AM_STAGING_DIR, STAGE_DIR);
    stageDir = TezCommonUtils.getTezBaseStagingPath(conf);
    Assert.assertEquals(stageDir.toString(), RESOLVED_STAGE_DIR);
}

---

From source file:org.apache.tez.dag.app.dag.impl.TestRootInputVertexManager.java

License:Apache License

static RootInputVertexManager createRootInputVertexManager(Configuration conf,
        VertexManagerPluginContext context, Float min, Float max) {
    if (min != null) {
        conf.setFloat(TEZ_ROOT_INPUT_VERTEX_MANAGER_MIN_SRC_FRACTION, min);
    } else {//  www  .  j av  a2 s .c  o m
        conf.unset(TEZ_ROOT_INPUT_VERTEX_MANAGER_MIN_SRC_FRACTION);
    }
    if (max != null) {
        conf.setFloat(TEZ_ROOT_INPUT_VERTEX_MANAGER_MAX_SRC_FRACTION, max);
    } else {
        conf.unset(TEZ_ROOT_INPUT_VERTEX_MANAGER_MAX_SRC_FRACTION);
    }
    if (max != null || min != null) {
        conf.setBoolean(TEZ_ROOT_INPUT_VERTEX_MANAGER_ENABLE_SLOW_START, true);
    }
    UserPayload payload;
    try {
        payload = TezUtils.createUserPayloadFromConf(conf);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    when(context.getUserPayload()).thenReturn(payload);
    RootInputVertexManager manager = new RootInputVertexManager(context);
    manager.initialize();
    return manager;
}

---

From source file:org.apache.tez.dag.library.vertexmanager.TestShuffleVertexManagerUtils.java

License:Apache License

static ShuffleVertexManager createShuffleVertexManager(Configuration conf, VertexManagerPluginContext context,
        Boolean enableAutoParallelism, Long desiredTaskInputSize, Float min, Float max) {
    if (min != null) {
        conf.setFloat(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION, min);
    } else {/*from   w w w. j av  a 2  s.c  o m*/
        conf.unset(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION);
    }
    if (max != null) {
        conf.setFloat(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION, max);
    } else {
        conf.unset(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION);
    }
    if (enableAutoParallelism != null) {
        conf.setBoolean(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_ENABLE_AUTO_PARALLEL,
                enableAutoParallelism);
    }
    if (desiredTaskInputSize != null) {
        conf.setLong(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_DESIRED_TASK_INPUT_SIZE,
                desiredTaskInputSize);
    }
    UserPayload payload;
    try {
        payload = TezUtils.createUserPayloadFromConf(conf);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    when(context.getUserPayload()).thenReturn(payload);
    ShuffleVertexManager manager = new ShuffleVertexManager(context);
    manager.initialize();
    return manager;
}

---

• «
• 1
• 2
• 3
• 4
• »