Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.lens.server; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; import java.io.File; import java.io.FileNotFoundException; import java.util.*; import javax.ws.rs.client.Entity; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.GenericType; import javax.ws.rs.core.MediaType; import org.apache.lens.api.APIResult; import org.apache.lens.api.LensConf; import org.apache.lens.api.LensSessionHandle; import org.apache.lens.api.query.LensQuery; import org.apache.lens.api.query.QueryHandle; import org.apache.lens.api.query.QueryStatus; import org.apache.lens.api.result.LensAPIResult; import org.apache.lens.server.api.LensConfConstants; import org.apache.commons.io.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; import org.glassfish.jersey.media.multipart.FormDataBodyPart; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.glassfish.jersey.media.multipart.FormDataMultiPart; import lombok.extern.slf4j.Slf4j; /** * The Class LensServerTestUtil. */ @Slf4j public final class LensServerTestUtil { public static final String DB_WITH_JARS = "test_db_static_jars"; public static final String DB_WITH_JARS_2 = "test_db_static_jars_2"; private LensServerTestUtil() { } /** * Creates the table. * * @param tblName the tbl name * @param parent the parent * @param lensSessionId the lens session id * @param schemaStr the schema string, with surrounding parenthesis. * @throws InterruptedException the interrupted exception */ public static void createTable(String tblName, WebTarget parent, LensSessionHandle lensSessionId, String schemaStr, MediaType mt) throws InterruptedException { LensConf conf = new LensConf(); conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); final WebTarget target = parent.path("queryapi/queries"); final FormDataMultiPart mp = new FormDataMultiPart(); String createTable = "CREATE TABLE IF NOT EXISTS " + tblName + schemaStr; mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId, mt)); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), createTable)); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "execute")); mp.bodyPart( new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf, mt)); final QueryHandle handle = target.request(mt).post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), new GenericType<LensAPIResult<QueryHandle>>() { }).getData(); // wait till the query finishes LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request(mt) .get(LensQuery.class); QueryStatus stat = ctx.getStatus(); while (!stat.finished()) { ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request(mt) .get(LensQuery.class); stat = ctx.getStatus(); Thread.sleep(1000); } final String debugHelpMsg = "Query Handle:" + ctx.getQueryHandleString(); assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL, debugHelpMsg); assertTrue(ctx.getSubmissionTime() > 0, debugHelpMsg); assertTrue(ctx.getLaunchTime() > 0, debugHelpMsg); assertTrue(ctx.getDriverStartTime() > 0, debugHelpMsg); assertTrue(ctx.getDriverFinishTime() > 0, debugHelpMsg); assertTrue(ctx.getFinishTime() > 0, debugHelpMsg); } public static void createTable(String tblName, WebTarget parent, LensSessionHandle lensSessionId, MediaType mt) throws InterruptedException { createTable(tblName, parent, lensSessionId, "(ID INT, IDSTR STRING)", mt); } public static void loadData(String tblName, final String testDataFile, WebTarget parent, LensSessionHandle lensSessionId, MediaType mt) throws InterruptedException { LensConf conf = new LensConf(); conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); final WebTarget target = parent.path("queryapi/queries"); final FormDataMultiPart mp = new FormDataMultiPart(); String dataLoad = "LOAD DATA LOCAL INPATH '" + testDataFile + "' OVERWRITE INTO TABLE " + tblName; mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId, mt)); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), dataLoad)); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "execute")); mp.bodyPart( new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf, mt)); final QueryHandle handle = target.request(mt).post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), new GenericType<LensAPIResult<QueryHandle>>() { }).getData(); // wait till the query finishes LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request(mt) .get(LensQuery.class); QueryStatus stat = ctx.getStatus(); while (!stat.finished()) { ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request(mt) .get(LensQuery.class); stat = ctx.getStatus(); Thread.sleep(1000); } assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); } /** * Load data. * * @param tblName the tbl name * @param testDataFile the test data file * @param parent the parent * @param lensSessionId the lens session id * @throws InterruptedException the interrupted exception */ public static void loadDataFromClasspath(String tblName, final String testDataFile, WebTarget parent, LensSessionHandle lensSessionId, MediaType mt) throws InterruptedException { String absolutePath = LensServerTestUtil.class.getClassLoader().getResource(testDataFile).getPath(); loadData(tblName, absolutePath, parent, lensSessionId, mt); } /** * Drop table. * * @param tblName the tbl name * @param parent the parent * @param lensSessionId the lens session id * @throws InterruptedException the interrupted exception */ public static void dropTable(String tblName, WebTarget parent, LensSessionHandle lensSessionId, MediaType mt) throws InterruptedException { LensConf conf = new LensConf(); conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); dropTableWithConf(tblName, parent, lensSessionId, conf, mt); } /** * Drop table with conf passed. * * @param tblName the tbl name * @param parent the parent * @param lensSessionId the lens session id * @param conf the query conf * * @throws InterruptedException */ public static void dropTableWithConf(String tblName, WebTarget parent, LensSessionHandle lensSessionId, LensConf conf, MediaType mt) throws InterruptedException { final WebTarget target = parent.path("queryapi/queries"); final FormDataMultiPart mp = new FormDataMultiPart(); String createTable = "DROP TABLE IF EXISTS " + tblName; mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId, mt)); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), createTable)); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "execute")); mp.bodyPart( new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf, mt)); final QueryHandle handle = target.request(mt).post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), new GenericType<LensAPIResult<QueryHandle>>() { }).getData(); // wait till the query finishes LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request(mt) .get(LensQuery.class); QueryStatus stat = ctx.getStatus(); while (!stat.finished()) { ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request(mt) .get(LensQuery.class); stat = ctx.getStatus(); Thread.sleep(1000); } assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); } /** * Creates the hive table. * * @param tableName the table name * @throws HiveException the hive exception */ public static void createHiveTable(String tableName, Map<String, String> parameters) throws HiveException { List<FieldSchema> columns = new ArrayList<FieldSchema>(); columns.add(new FieldSchema("col1", "string", "")); List<FieldSchema> partCols = new ArrayList<FieldSchema>(); partCols.add(new FieldSchema("pcol1", "string", "")); Map<String, String> params = new HashMap<String, String>(); params.put("test.hive.table.prop", "tvalue"); if (null != parameters && !parameters.isEmpty()) { params.putAll(parameters); } Table tbl = Hive.get().newTable(tableName); tbl.setTableType(TableType.MANAGED_TABLE); tbl.getTTable().getSd().setCols(columns); tbl.setPartCols(partCols); tbl.getTTable().getParameters().putAll(params); Hive.get().createTable(tbl); } /** * Drop hive table. * * @param tableName the table name * @throws HiveException the hive exception */ public static void dropHiveTable(String tableName) throws HiveException { Hive.get().dropTable(tableName); } public static void createTestDatabaseResources(String[] testDatabases, HiveConf conf) throws Exception { File srcJarDir = new File("target/testjars/"); if (!srcJarDir.exists()) { // nothing to setup return; } File resDir = new File("target/resources"); if (!resDir.exists()) { resDir.mkdir(); } // Create databases and resource dirs Hive hive = Hive.get(conf); File testJarFile = new File("target/testjars/test.jar"); File serdeJarFile = new File("target/testjars/serde.jar"); for (String db : testDatabases) { Database database = new Database(); database.setName(db); hive.createDatabase(database, true); File dbDir = new File(resDir, db); if (!dbDir.exists()) { dbDir.mkdir(); } // Add a jar in the directory try { String[] jarOrder = { "x_" + db + ".jar", "y_" + db + ".jar", "z_" + db + ".jar", "serde.jar", }; // Jar order is -> z, y, x, File listing order is x, y, z // We are explicitly specifying jar order FileUtils.writeLines(new File(dbDir, "jar_order"), Arrays.asList(jarOrder[2], jarOrder[1], jarOrder[0], jarOrder[3])); FileUtils.copyFile(testJarFile, new File(dbDir, jarOrder[0])); FileUtils.copyFile(testJarFile, new File(dbDir, jarOrder[1])); FileUtils.copyFile(testJarFile, new File(dbDir, jarOrder[2])); FileUtils.copyFile(serdeJarFile, new File(dbDir, jarOrder[3])); } catch (FileNotFoundException fnf) { log.error("File not found.", fnf); } } } public static LensSessionHandle openSession(WebTarget target, final String userName, final String passwd, final LensConf conf, MediaType mt) { final WebTarget sessionTarget = target.path("session"); final FormDataMultiPart mp = new FormDataMultiPart(); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("username").build(), userName)); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("password").build(), passwd)); mp.bodyPart(new FormDataBodyPart( FormDataContentDisposition.name("sessionconf").fileName("sessionconf").build(), conf, mt)); return sessionTarget.request(mt).post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), LensSessionHandle.class); } public static void addResource(WebTarget target, final LensSessionHandle lensSessionHandle, final String resourceType, final String resourcePath, MediaType mt) { final WebTarget resourceTarget = target.path("session/resources"); final FormDataMultiPart mp = new FormDataMultiPart(); mp.bodyPart( new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionHandle, mt)); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("type").build(), resourceType)); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("path").build(), resourcePath)); APIResult result = resourceTarget.path("add").request(mt) .put(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), APIResult.class); if (!result.getStatus().equals(APIResult.Status.SUCCEEDED)) { throw new RuntimeException("Could not add resource:" + result); } } }