Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package com.aliyun.odps.ship.common; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.nio.charset.Charset; import java.nio.charset.IllegalCharsetNameException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Properties; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import com.aliyun.odps.PartitionSpec; import com.aliyun.openservices.odps.console.ODPSConsoleException; public class OptionsBuilder { public static void buildUploadOption(String[] args) throws ParseException, IOException, ODPSConsoleException { DshipContext.INSTANCE.clear(); CommandLineParser parser = new GnuParser(); Options opts = getUploadOptions(); CommandLine line = parser.parse(opts, args); initContext(); // load context from config file loadConfig(); processOptions(line); processArgs(line.getArgs(), true); checkParameters("upload"); setContextValue(Constants.COMMAND, buildCommand(args)); setContextValue(Constants.COMMAND_TYPE, "upload"); } public static void buildDownloadOption(String[] args) throws ParseException, IOException, ODPSConsoleException { DshipContext.INSTANCE.clear(); CommandLineParser parser = new GnuParser(); Options opts = getDownloadOptions(); CommandLine line = parser.parse(opts, args); initContext(); // load context from config file loadConfig(); processOptions(line); processArgs(line.getArgs(), false); setContextValue(Constants.COMMAND, buildCommand(args)); setContextValue(Constants.COMMAND_TYPE, "download"); checkParameters("download"); } private static String buildCommand(String[] args) { StringBuilder cl = new StringBuilder(); for (String a : args) { cl.append(a + " "); } return cl.toString(); } public static void buildResumeOption(String[] args) throws ParseException { DshipContext.INSTANCE.clear(); CommandLineParser parser = new GnuParser(); Options opts = getResumeOptions(); CommandLine line = parser.parse(opts, args); if (line.hasOption("force")) { DshipContext.INSTANCE.put("resume-force", "true"); } String[] remains = line.getArgs(); if (remains.length == 2) { DshipContext.INSTANCE.put(Constants.SESSION_ID, remains[1]); } else if (remains.length > 2) { throw new IllegalArgumentException("Unknown command\nType 'tunnel help resume' for usage."); } } public static void buildShowOption(String[] args) throws ParseException { DshipContext.INSTANCE.clear(); CommandLineParser parser = new GnuParser(); Options opts = getShowOptions(); CommandLine line = parser.parse(opts, args); String[] remains = line.getArgs(); if (remains.length == 3) { DshipContext.INSTANCE.put(Constants.SHOW_COMMAND, getShowCmd(remains[1])); DshipContext.INSTANCE.put(Constants.SESSION_ID, remains[2]); } else if (remains.length == 2) { DshipContext.INSTANCE.put(Constants.SHOW_COMMAND, getShowCmd(remains[1])); } else { throw new IllegalArgumentException("Unknown command\nType 'tunnel help show' for usage."); } DshipContext.INSTANCE.put("number", line.getOptionValue("number")); checkShowCommandParameters(); } private static String getShowCmd(String cmd) { if (cmd.equals("h")) { return "history"; } else if (cmd.equals("l")) { return "log"; } else if (cmd.equals("b")) { return "bad"; } return cmd; } public static void buildPurgeOption(String[] args) throws ParseException { DshipContext.INSTANCE.clear(); CommandLineParser parser = new GnuParser(); Options opts = getPurgeOptions(); CommandLine line = parser.parse(opts, args); String[] remains = line.getArgs(); if (remains.length == 1) { DshipContext.INSTANCE.put(Constants.PURGE_NUMBER, Constants.DEFAULT_PURGE_NUMBER); } else if (remains.length == 2) { // check parameter try { Integer.valueOf(remains[1]); } catch (NumberFormatException e) { throw new IllegalArgumentException("Illegal number\nType 'tunnel help purge' for usage."); } DshipContext.INSTANCE.put(Constants.PURGE_NUMBER, remains[1]); } else if (remains.length > 2) { throw new IllegalArgumentException("Unknown command\nType 'tunnel help purge' for usage."); } } public static void buildHelpOption(String[] args) throws ParseException { DshipContext.INSTANCE.clear(); CommandLineParser parser = new GnuParser(); Options opts = getGlobalOptions(); CommandLine line = parser.parse(opts, args); String[] remains = line.getArgs(); if (remains.length > 2) { throw new IllegalArgumentException( "Unknown command: too many subcommands.\nType 'tunnel help' for usage."); } if (remains.length > 1) { try { CommandType.fromString(remains[1]); DshipContext.INSTANCE.put(Constants.HELP_SUBCOMMAND, remains[1]); } catch (IllegalArgumentException e) { throw new IllegalArgumentException( "Unknown command: " + remains[1] + "\nType 'tunnel help' for usage."); } } } public static void checkParameters(String type) { String table = DshipContext.INSTANCE.get(Constants.TABLE); if (table == null || "".equals(table.trim())) { throw new IllegalArgumentException("Table is null.\nType 'tunnel help " + type + "' for usage."); } String fd = DshipContext.INSTANCE.get(Constants.FIELD_DELIMITER); if (fd == null || fd.length() == 0) { throw new IllegalArgumentException("Field delimiter is null."); } String rd = DshipContext.INSTANCE.get(Constants.RECORD_DELIMITER); if (rd == null || rd.length() == 0) { throw new IllegalArgumentException("Record delimiter is null."); } if (fd.contains(rd)) { throw new IllegalArgumentException( "Field delimiter can not include record delimiter.\nType 'tunnel help " + type + "' for usage."); } boolean isc = false; String c = DshipContext.INSTANCE.get(Constants.CHARSET); try { isc = Charset.isSupported(c); } catch (IllegalCharsetNameException e) { //ignore bad charset name } isc = isc || Constants.IGNORE_CHARSET.equals(c); if (c == null || c.isEmpty() || !isc) { throw new IllegalArgumentException( "Unsupported encoding: '" + c + "'\nType 'tunnel help " + type + "' for usage."); } try { new SimpleDateFormat(DshipContext.INSTANCE.get(Constants.DATE_FORMAT_PATTERN)); } catch (IllegalArgumentException e) { throw new IllegalArgumentException("Unsupported date format pattern '" + DshipContext.INSTANCE.get(Constants.DATE_FORMAT_PATTERN) + "'"); } String sct = DshipContext.INSTANCE.get(Constants.SESSION_CREATE_TIME); if (sct == null) { throw new IllegalArgumentException(Constants.ERROR_INDICATOR + "create time is null."); } String cp = DshipContext.INSTANCE.get(Constants.COMPRESS); if (cp == null) { throw new IllegalArgumentException(Constants.ERROR_INDICATOR + "compress info is null."); } int threads; try { threads = Integer.parseInt(DshipContext.INSTANCE.get(Constants.THREADS)); } catch (IllegalArgumentException e) { throw new IllegalArgumentException( Constants.THREADS + " " + DshipContext.INSTANCE.get(Constants.THREADS) + " Invalid."); } if (threads <= 0) { throw new IllegalArgumentException(Constants.THREADS + " argument must > 0."); } if (threads > 1 && "true".equalsIgnoreCase(DshipContext.INSTANCE.get(Constants.HEADER))) { throw new IllegalArgumentException("Do not support write header in multi-threads."); } if ("download".equals(type)) { String limit = DshipContext.INSTANCE.get(Constants.LIMIT); if (limit != null) { try { if (Long.valueOf(limit) <= 0) { throw new IllegalArgumentException(Constants.LIMIT + " argument must > 0."); } } catch (NumberFormatException e) { throw new IllegalArgumentException(Constants.LIMIT + " " + limit + " Invalid."); } } } if ("upload".equals(type)) { String scan = DshipContext.INSTANCE.get(Constants.SCAN); String dbr = DshipContext.INSTANCE.get(Constants.DISCARD_BAD_RECORDS); if (scan == null || !(scan.equals("true") || scan.equals("false") || scan.equals("only"))) { throw new IllegalArgumentException("-scan, expected:(true|false|only), actual: '" + scan + "'\nType 'tunnel help " + type + "' for usage."); } if (dbr == null || !(dbr.equals("true") || dbr.equals("false"))) { throw new IllegalArgumentException( "Invalid parameter : discard bad records expected 'true' or 'false', found '" + dbr + "'\nType 'tunnel help " + type + "' for usage."); } String bs = DshipContext.INSTANCE.get(Constants.BLOCK_SIZE); String mbr = DshipContext.INSTANCE.get(Constants.MAX_BAD_RECORDS); if (bs != null) { try { Long.valueOf(bs); } catch (NumberFormatException e) { throw new IllegalArgumentException("Illegal number 'block-size', please check config file."); } } if (mbr != null) { try { Long.valueOf(mbr); } catch (NumberFormatException e) { throw new IllegalArgumentException( "Illegal number 'max-bad-records', please check config file."); } } String path = DshipContext.INSTANCE.get(Constants.RESUME_PATH); File sFile = new File(path); if (!sFile.exists()) { throw new IllegalArgumentException( "upload File not found: '" + path + "'\nType 'tunnel help " + type + "' for usage."); } } } private static void checkShowCommandParameters() { String cmd = DshipContext.INSTANCE.get(Constants.SHOW_COMMAND); String number = DshipContext.INSTANCE.get("number"); String sid = DshipContext.INSTANCE.get(Constants.SESSION_ID); if (!(cmd.equals("log") || cmd.equals("bad") || cmd.equals("history"))) { throw new IllegalArgumentException("Unknown command\nType 'tunnel help show' for usage."); } if ((cmd.equals("log") || cmd.equals("bad")) && (number != null)) { throw new IllegalArgumentException("Unknown command\nType 'tunnel help show' for usage."); } if (cmd.equals("history") && sid != null) { throw new IllegalArgumentException("Unknown command\nType 'tunnel help show' for usage."); } if (number != null) { try { Integer.valueOf(number); } catch (NumberFormatException e) { throw new IllegalArgumentException("Illegal number\nType 'tunnel help show' for usage."); } } } private static void initContext() { // set default value setContextValue(Constants.CHARSET, Constants.IGNORE_CHARSET); setContextValue(Constants.FIELD_DELIMITER, Constants.DEFAULT_FIELD_DELIMITER); setContextValue(Constants.RECORD_DELIMITER, Constants.DEFAULT_RECORD_DELIMITER); setContextValue(Constants.DISCARD_BAD_RECORDS, Constants.DEFAULT_DISCARD_BAD_RECORDS); setContextValue(Constants.DATE_FORMAT_PATTERN, Constants.DEFAULT_DATE_FORMAT_PATTERN); setContextValue(Constants.NULL_INDICATOR, Constants.DEFAULT_NULL_INDICATOR); setContextValue(Constants.SCAN, Constants.DEFAULT_SCAN); setContextValue(Constants.HEADER, Constants.DEFAULT_HEADER); setContextValue(Constants.RESUME_BLOCK_ID, "1"); setContextValue(Constants.SESSION_CREATE_TIME, String.valueOf(System.currentTimeMillis())); setContextValue(Constants.COMPRESS, "true"); setContextValue(Constants.THREADS, String.valueOf(Constants.DEFAULT_THREADS)); } private static void processOptions(CommandLine line) { // set context value from options Option[] ops = line.getOptions(); for (Option op : ops) { String v = removeQuote(op.getValue()); if (Constants.FIELD_DELIMITER.equals(op.getLongOpt()) || Constants.RECORD_DELIMITER.equals(op.getLongOpt())) { setContextValue(op.getLongOpt(), processDelimiter(v)); } else { setContextValue(op.getLongOpt(), v); } } } private static String processDelimiter(String delimiter) { if (delimiter != null) { return delimiter.replaceAll("\\\\r", "\r").replaceAll("\\\\n", "\n").replaceAll("\\\\t", "\t"); } return null; } private static String removeQuote(String in) { if (in == null || in.length() < 2) { return in; } if ((in.startsWith("\"") && in.endsWith("\"")) || (in.startsWith("'") && (in.endsWith("'")))) { return in.substring(1, in.length() - 1); } return in; } private static void processArgs(String[] remains, boolean isUpload) throws IOException { String path = null; String partition = null; String table = null; if (remains.length == 3) { path = isUpload ? remains[1] : remains[2]; String desc = isUpload ? remains[2] : remains[1]; String[] ds = desc.split("/"); String[] tp = ds[0].split("\\."); if (tp.length == 1) { table = tp[0]; } else if (tp.length == 2) { table = tp[1]; setContextValue(Constants.TABLE_PROJECT, tp[0]); } else { throw new IllegalArgumentException("Invalid parameter: project.table \nType 'tunnel help " + (isUpload ? "upload" : "download") + "' for usage."); } if (ds.length == 2) { partition = new PartitionSpec(ds[1]).toString(); } else if (ds.length > 2) { throw new IllegalArgumentException("Invalid parameter: project.table/partition\nType 'tunnel help " + (isUpload ? "upload" : "download") + "' for usage."); } } else { throw new IllegalArgumentException("Unrecognized command\nType 'tunnel help " + (isUpload ? "upload" : "download") + "' for usage."); } setContextValue(Constants.TABLE, table); setContextValue(Constants.PARTITION_SPEC, partition); setContextValue(Constants.RESUME_PATH, path); } private static void loadConfig() throws IOException, ODPSConsoleException { String cf = DshipContext.INSTANCE.getExecutionContext().getConfigFile(); if (cf == null) { return; } File file = new File(cf); if (!file.exists()) { return; } FileInputStream cfIns = new FileInputStream(file); Properties properties = new Properties(); properties.load(cfIns); for (Object key : properties.keySet()) { String k = key.toString(); String v = (String) properties.get(key); setContextValue(k.toLowerCase(), v); } cfIns.close(); } private static void setContextValue(String key, String value) { if (value != null) { if (!key.equals(Constants.FIELD_DELIMITER) && !key.equals(Constants.RECORD_DELIMITER) && !key.equals(Constants.PARTITION_SPEC)) { value = value.trim(); } DshipContext.INSTANCE.put(key, value); } } public static Options getGlobalOptions() { Options opts = new Options(); opts.addOption(OptionBuilder.withLongOpt(Constants.CHARSET) .withDescription("specify file charset, default " + Constants.IGNORE_CHARSET + ". set " + Constants.IGNORE_CHARSET + " to download raw data") .hasArg().withArgName("ARG").create("c")); opts.addOption(OptionBuilder.withLongOpt(Constants.FIELD_DELIMITER) .withDescription("specify field delimiter, default " + Util.toHumanReadableString(Constants.DEFAULT_FIELD_DELIMITER)) .hasArg().withArgName("ARG").create("fd")); opts.addOption(OptionBuilder.withLongOpt(Constants.RECORD_DELIMITER) .withDescription("specify record delimiter, default " + Util.toHumanReadableString(Constants.DEFAULT_RECORD_DELIMITER)) .hasArg().withArgName("ARG").create("rd")); opts.addOption(OptionBuilder.withLongOpt(Constants.DATE_FORMAT_PATTERN) .withDescription("specify date format pattern, default " + Constants.DEFAULT_DATE_FORMAT_PATTERN) .hasArg().withArgName("ARG").create("dfp")); opts.addOption(OptionBuilder.withLongOpt(Constants.NULL_INDICATOR) .withDescription("specify null indicator string, default " + Util.toHumanReadableString(Constants.DEFAULT_NULL_INDICATOR)) .hasArg().withArgName("ARG").create("ni")); opts.addOption(OptionBuilder.withLongOpt(Constants.TIME_ZONE) .withDescription( "time zone, default local timezone: " + Calendar.getInstance().getTimeZone().getID()) .hasArg().withArgName("ARG").create("tz")); opts.addOption(OptionBuilder.withLongOpt(Constants.COMPRESS).withDescription("compress, default true") .hasArg().withArgName("ARG").create("cp")); opts.addOption(OptionBuilder.withLongOpt(Constants.HEADER) .withDescription("if local file should have table header, default " + Constants.DEFAULT_HEADER) .hasArg().withArgName("ARG").create("h")); opts.addOption(OptionBuilder.withLongOpt(Constants.SESSION_DIR) .withDescription("set session dir, default " + Constants.DEFAULT_SESSION_DIR).hasArg() .withArgName("ARG").create("sd")); opts.addOption(OptionBuilder.withLongOpt(Constants.THREADS) .withDescription("number of threads, default " + Constants.DEFAULT_THREADS).hasArg() .withArgName("ARG").create()); return opts; } public static Options getUploadOptions() { Options opts = getGlobalOptions(); opts.addOption(OptionBuilder.withLongOpt(Constants.DISCARD_BAD_RECORDS).withDescription( "specify discard bad records action(true|false), default " + Constants.DEFAULT_DISCARD_BAD_RECORDS) .hasArg().withArgName("ARG").create("dbr")); opts.addOption(OptionBuilder.withLongOpt(Constants.SCAN) .withDescription("specify scan file action(true|false|only), default " + Constants.DEFAULT_SCAN) .hasArg().withArgName("ARG").create("s")); opts.addOption(OptionBuilder.withLongOpt(Constants.BLOCK_SIZE) .withDescription("block size in MiB, default " + Constants.DEFAULT_BLOCK_SIZE).hasArg() .withArgName("ARG").create("bs")); opts.addOption(OptionBuilder.withLongOpt(Constants.MAX_BAD_RECORDS) .withDescription("max bad records, default " + Constants.DEFAULT_BAD_RECORDS).hasArg() .withArgName("ARG").create("mbr")); return opts; } public static Options getDownloadOptions() { Options opts = getGlobalOptions(); opts.addOption(OptionBuilder.withLongOpt(Constants.LIMIT) .withDescription("specify the number of records to download").hasArg().withArgName("ARG").create()); return opts; } public static Options getResumeOptions() { Options opts = getGlobalOptions(); opts.addOption(OptionBuilder.withLongOpt("force").withDescription("force resume").create("f")); return opts; } public static Options getShowOptions() { Options opts = getGlobalOptions(); opts.addOption(OptionBuilder.withLongOpt("number").withDescription("lines").hasArg().withArgName("ARG") .create("n")); return opts; } public static Options getConfigOptions() { return getGlobalOptions(); } public static Options getPurgeOptions() { return getGlobalOptions(); } }