Example usage for org.apache.hadoop.conf Configuration setLong

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration setLong.

Prototype

public void setLong(String name, long value) 

Source Link

Document

Set the value of the name property to a long.

Usage

From source file:org.apache.jena.hadoop.rdf.io.input.AbstractNodeTupleInputFormatTests.java

License:Apache License

/**
 * Tests for input splitting// ww w . j  a  v a2  s .c  om
 * 
 * @throws IOException
 * @throws InterruptedException
 */
@Test
public final void split_input_03() throws IOException, InterruptedException {
    Assume.assumeTrue(this.canSplitInputs());

    Configuration config = this.prepareConfiguration();
    config.setBoolean(RdfIOConstants.INPUT_IGNORE_BAD_TUPLES, false);
    config.setLong(NLineInputFormat.LINES_PER_MAP, 100);
    Assert.assertEquals(Integer.MAX_VALUE, config.getInt(HadoopIOConstants.MAX_LINE_LENGTH, Integer.MAX_VALUE));
    this.testSplitInputs(config, new File[] { large }, 100, LARGE_SIZE);
}

---

From source file:org.apache.jena.hadoop.rdf.io.output.trig.BatchedTriGOutputTest.java

License:Apache License

@Override
protected Configuration prepareConfiguration() {
    Configuration config = super.prepareConfiguration();
    config.setLong(RdfIOConstants.OUTPUT_BATCH_SIZE, this.batchSize);
    return config;
}

---

From source file:org.apache.kylin.storage.hbase.cube.v1.filter.TestFuzzyRowFilterV2EndToEnd.java

License:Apache License

/**
 * @throws java.lang.Exception//from www. j  a  v  a  2s  .  c  o  m
 */
@BeforeClass
public static void setUpBeforeClass() throws Exception {
    Configuration conf = TEST_UTIL.getConfiguration();
    conf.setInt("hbase.client.scanner.caching", 1000);
    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName());
    // set no splits
    conf.setLong(HConstants.HREGION_MAX_FILESIZE, ((long) 1024) * 1024 * 1024 * 10);

    TEST_UTIL.startMiniCluster();
}

---

From source file:org.apache.lens.cube.parse.TestTimeRangeResolver.java

License:Apache License

@Test
public void testCustomNow() throws Exception {
    Configuration conf = getConf();
    DateTime dt = new DateTime(1990, 3, 23, 12, 0, 0, 0);
    conf.setLong(LensConfConstants.QUERY_CURRENT_TIME_IN_MILLIS, dt.getMillis());
    CubeQueryContext ctx = rewriteCtx(//from  w  w  w .ja v  a2 s. c  o m
            "select msr12 from basecube where time_range_in(d_time, 'now.day-275days','now')", conf);
    TimeRange timeRange = ctx.getTimeRanges().get(0);
    // Month starts from zero.
    Calendar from = new GregorianCalendar(1989, 5, 21, 0, 0, 0);
    assertEquals(timeRange.getFromDate(), from.getTime());
    assertEquals(timeRange.getToDate(), dt.toDate());
}

---

From source file:org.apache.lens.driver.hive.TestRemoteHiveDriver.java

License:Apache License

/**
 * Test multi thread client./*from  w  w  w . j a  v a2  s.  c  o  m*/
 *
 * @throws Exception the exception
 */
@Test
public void testMultiThreadClient() throws Exception {
    log.info("@@ Starting multi thread test");
    SessionState.get().setCurrentDatabase(dataBase);
    final SessionState state = SessionState.get();
    // Launch two threads
    createTestTable("test_multithreads");
    Configuration thConf = new Configuration(driverConf);
    thConf.setLong(HiveDriver.HS2_CONNECTION_EXPIRY_DELAY, 10000);
    final HiveDriver thrDriver = new HiveDriver();
    thrDriver.configure(thConf, "hive", "hive1");
    QueryContext ctx = createContext("USE " + dataBase, queryConf, thrDriver);
    thrDriver.execute(ctx);

    // Launch a select query
    final int QUERIES = 5;
    int launchedQueries = 0;
    final int THREADS = 5;
    final long POLL_DELAY = 500;
    List<Thread> thrs = new ArrayList<Thread>();
    List<QueryContext> queries = new ArrayList<>();
    final AtomicInteger errCount = new AtomicInteger();
    for (int q = 0; q < QUERIES; q++) {
        final QueryContext qctx;
        try {
            qctx = createContext("SELECT * FROM test_multithreads", queryConf, thrDriver);
            thrDriver.executeAsync(qctx);
            queries.add(qctx);
        } catch (LensException e) {
            errCount.incrementAndGet();
            log.info(q + " executeAsync error: " + e.getCause());
            continue;
        }
        log.info("@@ Launched query: " + q + " " + qctx.getQueryHandle());
        launchedQueries++;
        // Launch many threads to poll for status
        final QueryHandle handle = qctx.getQueryHandle();
        for (int i = 0; i < THREADS; i++) {
            int thid = q * THREADS + i;
            Thread th = new Thread(new Runnable() {
                @Override
                public void run() {
                    SessionState.setCurrentSessionState(state);
                    for (int i = 0; i < 1000; i++) {
                        try {
                            thrDriver.updateStatus(qctx);
                            if (qctx.getDriverStatus().isFinished()) {
                                log.info("@@ " + handle.getHandleId() + " >> "
                                        + qctx.getDriverStatus().getState());
                                break;
                            }
                            Thread.sleep(POLL_DELAY);
                        } catch (LensException e) {
                            log.error("Got Exception " + e.getCause(), e);
                            errCount.incrementAndGet();
                            break;
                        } catch (InterruptedException e) {
                            log.error("Encountred Interrupted exception", e);
                            break;
                        }
                    }
                }
            });
            thrs.add(th);
            th.setName("Poller#" + (thid));
            th.start();
        }
    }

    for (Thread th : thrs) {
        try {
            th.join(10000);
        } catch (InterruptedException e) {
            log.warn("Not ended yet: " + th.getName());
        }
    }
    for (QueryContext queryContext : queries) {
        thrDriver.closeQuery(queryContext.getQueryHandle());
    }
    Assert.assertEquals(0, thrDriver.getHiveHandleSize());
    log.info("@@ Completed all pollers. Total thrift errors: " + errCount.get());
    assertEquals(launchedQueries, QUERIES);
    assertEquals(thrs.size(), QUERIES * THREADS);
    assertEquals(errCount.get(), 0);
}

---

From source file:org.apache.lens.driver.hive.TestRemoteHiveDriver.java

License:Apache License

/**
 * Test hive driver persistence./*w ww  . j av  a2  s .  c  o  m*/
 *
 * @throws Exception the exception
 */
@Test
public void testHiveDriverPersistence() throws Exception {
    System.out.println("@@@@ start_persistence_test");
    Configuration driverConf = new Configuration(remoteConf);
    driverConf.addResource("drivers/hive/hive1/hivedriver-site.xml");
    driverConf.setLong(HiveDriver.HS2_CONNECTION_EXPIRY_DELAY, 10000);
    driverConf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, false);

    final HiveDriver oldDriver = new HiveDriver();
    oldDriver.configure(driverConf, "hive", "hive1");

    queryConf.setBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE, false);
    queryConf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false);
    QueryContext ctx = createContext("USE " + dataBase, queryConf, oldDriver);
    oldDriver.execute(ctx);
    Assert.assertEquals(0, oldDriver.getHiveHandleSize());

    String tableName = "test_hive_driver_persistence";

    // Create some ops with a driver
    String createTable = "CREATE TABLE IF NOT EXISTS " + tableName + "(ID STRING)";
    ctx = createContext(createTable, queryConf, oldDriver);
    oldDriver.execute(ctx);

    // Load some data into the table
    String dataLoad = "LOAD DATA LOCAL INPATH '" + TEST_DATA_FILE + "' OVERWRITE INTO TABLE " + tableName;
    ctx = createContext(dataLoad, queryConf, oldDriver);
    oldDriver.execute(ctx);

    queryConf.setBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE, true);
    queryConf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, true);
    // Fire two queries
    QueryContext ctx1 = createContext("SELECT * FROM " + tableName, queryConf, oldDriver);
    oldDriver.executeAsync(ctx1);
    QueryContext ctx2 = createContext("SELECT ID FROM " + tableName, queryConf, oldDriver);
    oldDriver.executeAsync(ctx2);
    Assert.assertEquals(2, oldDriver.getHiveHandleSize());

    byte[] ctx1bytes = persistContext(ctx1);
    byte[] ctx2bytes = persistContext(ctx2);

    // Write driver to stream
    ByteArrayOutputStream driverBytes = new ByteArrayOutputStream();
    ObjectOutputStream out = new ObjectOutputStream(driverBytes);
    try {
        oldDriver.writeExternal(out);
    } finally {
        out.close();
        driverBytes.close();
    }

    // Create another driver from the stream
    ByteArrayInputStream driverInput = new ByteArrayInputStream(driverBytes.toByteArray());
    HiveDriver newDriver = new HiveDriver();
    newDriver.readExternal(new ObjectInputStream(driverInput));
    newDriver.configure(driverConf, "hive", "hive1");
    driverInput.close();

    ctx1 = readContext(ctx1bytes, newDriver);
    ctx2 = readContext(ctx2bytes, newDriver);

    Assert.assertEquals(2, newDriver.getHiveHandleSize());

    validateExecuteAsync(ctx1, DriverQueryState.SUCCESSFUL, true, false, newDriver);
    validateExecuteAsync(ctx2, DriverQueryState.SUCCESSFUL, true, false, newDriver);
}

---

From source file:org.apache.lens.lib.query.TestFilePersistentFormatter.java

License:Apache License

/**
 * Test text file with zip formatter.// w  w w . j  a  va2 s. c om
 *
 * @throws IOException Signals that an I/O exception has occurred.
 */
@Test
public void testTextFileWithZipFormatter() throws IOException {
    Configuration conf = new Configuration();
    setConf(conf);
    conf.set("test.partfile.dir", partFileTextDir.toString());
    conf.set(LensConfConstants.QUERY_OUTPUT_FILE_EXTN, ".txt");
    conf.set(LensConfConstants.QUERY_OUTPUT_HEADER, "");
    conf.set(LensConfConstants.QUERY_OUTPUT_SERDE, LazySimpleSerDe.class.getCanonicalName());
    conf.setBoolean(LensConfConstants.RESULT_SPLIT_INTO_MULTIPLE, true);
    conf.setLong(LensConfConstants.RESULT_SPLIT_MULTIPLE_MAX_ROWS, 2L);
    testFormatter(conf, "UTF8", LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT, ".zip",
            getMockedResultSetWithoutComma());
    // validate rows
    List<String> actual = readZipOutputFile(new Path(formatter.getFinalOutputPath()), conf, "UTF-8");
    System.out.println("Actual rows:" + actual);
    Assert.assertEquals(actual, getExpectedTextRowsWithMultipleWithoutComma());
}

---

From source file:org.apache.lens.lib.query.TestFilePersistentFormatter.java

License:Apache License

/**
 * Test csv with zip formatter./*  w w  w.j  a v a  2s . c o m*/
 *
 * @throws IOException Signals that an I/O exception has occurred.
 */
@Test
public void testCSVWithZipFormatter() throws IOException {
    Configuration conf = new Configuration();
    setConf(conf);
    conf.set(LensConfConstants.QUERY_OUTPUT_HEADER, "");
    conf.setBoolean(LensConfConstants.RESULT_SPLIT_INTO_MULTIPLE, true);
    conf.setLong(LensConfConstants.RESULT_SPLIT_MULTIPLE_MAX_ROWS, 2L);
    testFormatter(conf, "UTF8", LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT, ".zip", getMockedResultSet());
    // validate rows
    List<String> actual = readZipOutputFile(new Path(formatter.getFinalOutputPath()), conf, "UTF-8");
    System.out.println("Actual rows:" + actual);
    Assert.assertEquals(actual, getExpectedCSVRowsWithMultiple());
}

---

From source file:org.apache.lens.lib.query.TestFilePersistentFormatter.java

License:Apache License

/**
 * Test zip csv files output path./* w  w  w. j a v a 2s  .c o m*/
 *
 * @throws IOException Signals that an I/O exception has occurred.
 */
@Test
public void testCSVZipFileOutputPath() throws IOException {
    Configuration conf = new Configuration();
    setConf(conf);
    conf.setBoolean(LensConfConstants.RESULT_SPLIT_INTO_MULTIPLE, true);
    conf.setLong(LensConfConstants.RESULT_SPLIT_MULTIPLE_MAX_ROWS, 2L);
    QueryContext ctx = createContext(conf, "Test.query_1 name");
    Path expectedFinalPath = new Path(LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT,
            "Test.query_1_name-" + ctx.getQueryHandle() + ".zip");
    FileSystem fs = expectedFinalPath.getFileSystem(conf);
    expectedFinalPath = expectedFinalPath.makeQualified(fs);
    validateFormatter(conf, "UTF8", LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT, ".zip",
            getMockedResultSetWithoutComma(), ctx, expectedFinalPath);
    ZipEntry ze = null;
    ZipInputStream zin = new ZipInputStream(fs.open(expectedFinalPath));
    int i = 0;
    while ((ze = zin.getNextEntry()) != null) {
        Assert.assertEquals(ze.getName(), "Test.query_1_name-" + ctx.getQueryHandle() + "_part-" + i + ".csv");
        i++;
        zin.closeEntry();
    }
    zin.close();
}

---

From source file:org.apache.lens.lib.query.TestFileSerdeFormatter.java

License:Apache License

/**
 * Test text file with zip formatter./* w  w  w  . j  a v  a2  s.  co  m*/
 *
 * @throws IOException Signals that an I/O exception has occurred.
 */
@Test
public void testTextFileWithZipFormatter() throws IOException {
    Configuration conf = new Configuration();
    setConf(conf);
    conf.set(LensConfConstants.QUERY_OUTPUT_FILE_EXTN, ".txt");
    conf.set(LensConfConstants.QUERY_OUTPUT_SERDE, LazySimpleSerDe.class.getCanonicalName());
    conf.setBoolean(LensConfConstants.RESULT_SPLIT_INTO_MULTIPLE, true);
    conf.setLong(LensConfConstants.RESULT_SPLIT_MULTIPLE_MAX_ROWS, 2L);
    testFormatter(conf, "UTF8", LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT, ".zip",
            getMockedResultSetWithoutComma());
    // validate rows
    List<String> actual = readZipOutputFile(new Path(formatter.getFinalOutputPath()), conf, "UTF-8");
    Assert.assertEquals(actual, getExpectedTextRowsWithMultipleWithoutComma());
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• »