Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.hive.hcatalog.cli; import java.io.BufferedReader; import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; import java.io.OutputStream; import java.io.PrintStream; import java.io.PrintWriter; import java.io.UnsupportedEncodingException; import java.util.ArrayList; import java.util.Properties; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.cli.Parser; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.common.LogUtils; import org.apache.hadoop.hive.common.LogUtils.LogInitializationException; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.processors.DfsProcessor; import org.apache.hadoop.hive.ql.processors.SetProcessor; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hive.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; import org.apache.hive.hcatalog.common.HCatConstants; import org.apache.hive.hcatalog.common.HCatUtil; public class HCatCli { private static Logger LOG = null; @SuppressWarnings("static-access") public static void main(String[] args) { try { LogUtils.initHiveLog4j(); } catch (LogInitializationException e) { } LOG = LoggerFactory.getLogger(HCatCli.class); CliSessionState ss = new CliSessionState(new HiveConf(SessionState.class)); ss.in = System.in; try { ss.out = new PrintStream(System.out, true, "UTF-8"); ss.err = new PrintStream(System.err, true, "UTF-8"); } catch (UnsupportedEncodingException e) { System.exit(1); } HiveConf conf = ss.getConf(); HiveConf.setVar(conf, ConfVars.SEMANTIC_ANALYZER_HOOK, HCatSemanticAnalyzer.class.getName()); String engine = HiveConf.getVar(conf, ConfVars.HIVE_EXECUTION_ENGINE); final String MR_ENGINE = "mr"; if (!MR_ENGINE.equalsIgnoreCase(engine)) { HiveConf.setVar(conf, ConfVars.HIVE_EXECUTION_ENGINE, MR_ENGINE); LOG.info("Forcing " + ConfVars.HIVE_EXECUTION_ENGINE + " to " + MR_ENGINE); } Options options = new Options(); // -e 'quoted-query-string' options.addOption(OptionBuilder.hasArg().withArgName("exec") .withDescription("hcat command given from command line").create('e')); // -f <query-file> options.addOption( OptionBuilder.hasArg().withArgName("file").withDescription("hcat commands in file").create('f')); // -g options.addOption(OptionBuilder.hasArg().withArgName("group") .withDescription("group for the db/table specified in CREATE statement").create('g')); // -p options.addOption(OptionBuilder.hasArg().withArgName("perms") .withDescription("permissions for the db/table specified in CREATE statement").create('p')); // -D options.addOption(OptionBuilder.hasArgs(2).withArgName("property=value").withValueSeparator() .withDescription("use hadoop value for given property").create('D')); // [-h|--help] options.addOption(new Option("h", "help", false, "Print help information")); Parser parser = new GnuParser(); CommandLine cmdLine = null; try { cmdLine = parser.parse(options, args); } catch (ParseException e) { printUsage(options, System.err); // Note, we print to System.err instead of ss.err, because if we can't parse our // commandline, we haven't even begun, and therefore cannot be expected to have // reasonably constructed or started the SessionState. System.exit(1); } // -D : process these first, so that we can instantiate SessionState appropriately. setConfProperties(conf, cmdLine.getOptionProperties("D")); // Now that the properties are in, we can instantiate SessionState. SessionState.start(ss); // -h if (cmdLine.hasOption('h')) { printUsage(options, ss.out); sysExit(ss, 0); } // -e String execString = (String) cmdLine.getOptionValue('e'); // -f String fileName = (String) cmdLine.getOptionValue('f'); if (execString != null && fileName != null) { ss.err.println("The '-e' and '-f' options cannot be specified simultaneously"); printUsage(options, ss.err); sysExit(ss, 1); } // -p String perms = (String) cmdLine.getOptionValue('p'); if (perms != null) { validatePermissions(ss, conf, perms); } // -g String grp = (String) cmdLine.getOptionValue('g'); if (grp != null) { conf.set(HCatConstants.HCAT_GROUP, grp); } // all done parsing, let's run stuff! if (execString != null) { sysExit(ss, processLine(execString)); } try { if (fileName != null) { sysExit(ss, processFile(fileName)); } } catch (FileNotFoundException e) { ss.err.println("Input file not found. (" + e.getMessage() + ")"); sysExit(ss, 1); } catch (IOException e) { ss.err.println("Could not open input file for reading. (" + e.getMessage() + ")"); sysExit(ss, 1); } // -h printUsage(options, ss.err); sysExit(ss, 1); } /** * Wrapper for System.exit that makes sure we close our SessionState * before we exit. This ignores any error generated by attempting to * close the session state, merely printing out the error. The return * code is not changed in such an occurrence because we want to retain * the success code of whatever command we already ran. */ private static void sysExit(SessionState ss, int retCode) { try { ss.close(); } catch (IOException e) { // If we got an error attempting to ss.close, then it's not likely that // ss.err is valid. So we're back to System.err. Also, we don't change // the return code, we simply log a warning, and return whatever return // code we expected to do already. System.err.println(e.getMessage()); e.printStackTrace(System.err); } System.exit(retCode); } private static void setConfProperties(HiveConf conf, Properties props) { for (java.util.Map.Entry<Object, Object> e : props.entrySet()) conf.set((String) e.getKey(), (String) e.getValue()); } private static int processLine(String line) { int ret = 0; String command = ""; for (String oneCmd : line.split(";")) { if (StringUtils.endsWith(oneCmd, "\\")) { command += StringUtils.chop(oneCmd) + ";"; continue; } else { command += oneCmd; } if (StringUtils.isBlank(command)) { continue; } ret = processCmd(command); command = ""; } return ret; } private static int processFile(String fileName) throws IOException { FileReader fileReader = null; BufferedReader reader = null; try { fileReader = new FileReader(fileName); reader = new BufferedReader(fileReader); String line; StringBuilder qsb = new StringBuilder(); while ((line = reader.readLine()) != null) { qsb.append(line + "\n"); } return (processLine(qsb.toString())); } finally { if (fileReader != null) { fileReader.close(); } if (reader != null) { reader.close(); } } } private static int processCmd(String cmd) { SessionState ss = SessionState.get(); long start = System.currentTimeMillis(); cmd = cmd.trim(); String firstToken = cmd.split("\\s+")[0].trim(); if (firstToken.equalsIgnoreCase("set")) { return new SetProcessor().run(cmd.substring(firstToken.length()).trim()).getResponseCode(); } else if (firstToken.equalsIgnoreCase("dfs")) { return new DfsProcessor(ss.getConf()).run(cmd.substring(firstToken.length()).trim()).getResponseCode(); } HCatDriver driver = new HCatDriver(); int ret = driver.run(cmd).getResponseCode(); if (ret != 0) { driver.close(); sysExit(ss, ret); } ArrayList<String> res = new ArrayList<String>(); try { while (driver.getResults(res)) { for (String r : res) { ss.out.println(r); } res.clear(); } } catch (IOException e) { ss.err.println("Failed with exception " + e.getClass().getName() + ":" + e.getMessage() + "\n" + org.apache.hadoop.util.StringUtils.stringifyException(e)); ret = 1; } catch (CommandNeedRetryException e) { ss.err.println("Failed with exception " + e.getClass().getName() + ":" + e.getMessage() + "\n" + org.apache.hadoop.util.StringUtils.stringifyException(e)); ret = 1; } int cret = driver.close(); if (ret == 0) { ret = cret; } long end = System.currentTimeMillis(); if (end > start) { double timeTaken = (end - start) / 1000.0; ss.err.println("Time taken: " + timeTaken + " seconds"); } return ret; } private static void printUsage(Options options, OutputStream os) { PrintWriter pw = new PrintWriter(os); new HelpFormatter().printHelp(pw, 2 * HelpFormatter.DEFAULT_WIDTH, "hcat { -e \"<query>\" | -f \"<filepath>\" } [ -g \"<group>\" ] [ -p \"<perms>\" ] [ -D\"<name>=<value>\" ]", null, options, HelpFormatter.DEFAULT_LEFT_PAD, HelpFormatter.DEFAULT_DESC_PAD, null, false); pw.flush(); } private static void validatePermissions(CliSessionState ss, HiveConf conf, String perms) { perms = perms.trim(); FsPermission fp = null; if (perms.matches("^\\s*([r,w,x,-]{9})\\s*$")) { fp = FsPermission.valueOf("d" + perms); } else if (perms.matches("^\\s*([0-7]{3})\\s*$")) { fp = new FsPermission(Short.decode("0" + perms)); } else { ss.err.println("Invalid permission specification: " + perms); sysExit(ss, 1); } if (!HCatUtil.validateMorePermissive(fp.getUserAction(), fp.getGroupAction())) { ss.err.println("Invalid permission specification: " + perms + " : user permissions must be more permissive than group permission "); sysExit(ss, 1); } if (!HCatUtil.validateMorePermissive(fp.getGroupAction(), fp.getOtherAction())) { ss.err.println("Invalid permission specification: " + perms + " : group permissions must be more permissive than other permission "); sysExit(ss, 1); } if ((!HCatUtil.validateExecuteBitPresentIfReadOrWrite(fp.getUserAction())) || (!HCatUtil.validateExecuteBitPresentIfReadOrWrite(fp.getGroupAction())) || (!HCatUtil.validateExecuteBitPresentIfReadOrWrite(fp.getOtherAction()))) { ss.err.println("Invalid permission specification: " + perms + " : permissions must have execute permissions if read or write permissions are specified "); sysExit(ss, 1); } conf.set(HCatConstants.HCAT_PERMS, "d" + fp.toString()); } }