Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.flink.api.java.utils; import org.apache.commons.cli.Option; import org.apache.commons.lang3.math.NumberUtils; import org.apache.flink.annotation.Public; import org.apache.flink.annotation.PublicEvolving; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.configuration.Configuration; import org.apache.flink.util.Preconditions; import org.apache.hadoop.util.GenericOptionsParser; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; import java.io.Serializable; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.Properties; /** * This class provides simple utility methods for reading and parsing program arguments from different sources */ @Public public class ParameterTool extends ExecutionConfig.GlobalJobParameters implements Serializable, Cloneable { private static final long serialVersionUID = 1L; protected static final String NO_VALUE_KEY = "__NO_VALUE_KEY"; protected static final String DEFAULT_UNDEFINED = "<undefined>"; // ------------------ Constructors ------------------------ /** * Returns {@link ParameterTool} for the given arguments. The arguments are keys followed by values. * Keys have to start with '-' or '--' * <p> * <strong>Example arguments:</strong> * --key1 value1 --key2 value2 -key3 value3 * * @param args Input array arguments * @return A {@link ParameterTool} */ public static ParameterTool fromArgs(String[] args) { Map<String, String> map = new HashMap<String, String>(args.length / 2); String key = null; String value = null; boolean expectValue = false; for (String arg : args) { // check for -- argument if (arg.startsWith("--")) { if (expectValue) { // we got into a new key, even though we were a value --> current key is one without value if (value != null) { throw new IllegalStateException("Unexpected state"); } map.put(key, NO_VALUE_KEY); // key will be overwritten in the next step } key = arg.substring(2); expectValue = true; } // check for - argument else if (arg.startsWith("-")) { // we are waiting for a value, so this is a - prefixed value (negative number) if (expectValue) { if (NumberUtils.isNumber(arg)) { // negative number value = arg; expectValue = false; } else { if (value != null) { throw new IllegalStateException("Unexpected state"); } // We waited for a value but found a new key. So the previous key doesnt have a value. map.put(key, NO_VALUE_KEY); key = arg.substring(1); expectValue = true; } } else { // we are not waiting for a value, so its an argument key = arg.substring(1); expectValue = true; } } else { if (expectValue) { value = arg; expectValue = false; } else { throw new RuntimeException("Error parsing arguments '" + Arrays.toString(args) + "' on '" + arg + "'. Unexpected value. Please prefix values with -- or -."); } } if (value == null && key == null) { throw new IllegalStateException("Value and key can not be null at the same time"); } if (key != null && value == null && !expectValue) { throw new IllegalStateException("Value expected but flag not set"); } if (key != null && value != null) { map.put(key, value); key = null; value = null; expectValue = false; } if (key != null && key.length() == 0) { throw new IllegalArgumentException( "The input " + Arrays.toString(args) + " contains an empty argument"); } if (key != null && !expectValue) { map.put(key, NO_VALUE_KEY); key = null; expectValue = false; } } if (key != null) { map.put(key, NO_VALUE_KEY); } return fromMap(map); } /** * Returns {@link ParameterTool} for the given {@link Properties} file * * @param path Path to the properties file * @return A {@link ParameterTool} * @throws IOException If the file does not exist * * @see Properties */ public static ParameterTool fromPropertiesFile(String path) throws IOException { File propertiesFile = new File(path); if (!propertiesFile.exists()) { throw new FileNotFoundException( "Properties file " + propertiesFile.getAbsolutePath() + " does not exist"); } Properties props = new Properties(); try (FileInputStream fis = new FileInputStream(propertiesFile)) { props.load(fis); } return fromMap((Map) props); } /** * Returns {@link ParameterTool} for the given map * * @param map A map of arguments. Both Key and Value have to be Strings * @return A {@link ParameterTool} */ public static ParameterTool fromMap(Map<String, String> map) { Preconditions.checkNotNull(map, "Unable to initialize from empty map"); return new ParameterTool(map); } /** * Returns {@link ParameterTool} from the system properties. * Example on how to pass system properties: * -Dkey1=value1 -Dkey2=value2 * * @return A {@link ParameterTool} */ public static ParameterTool fromSystemProperties() { return fromMap((Map) System.getProperties()); } /** * Returns {@link ParameterTool} for the arguments parsed by {@link GenericOptionsParser} * * @param args Input array arguments. It should be parsable by {@link GenericOptionsParser} * @return A {@link ParameterTool} * @throws IOException If arguments cannot be parsed by {@link GenericOptionsParser} * @see GenericOptionsParser * @deprecated Please use {@link org.apache.flink.hadoopcompatibility.HadoopUtils#paramsFromGenericOptionsParser(String[])} * from project flink-hadoop-compatibility */ @Deprecated @PublicEvolving public static ParameterTool fromGenericOptionsParser(String[] args) throws IOException { Option[] options = new GenericOptionsParser(args).getCommandLine().getOptions(); Map<String, String> map = new HashMap<String, String>(); for (Option option : options) { String[] split = option.getValue().split("="); map.put(split[0], split[1]); } return fromMap(map); } // ------------------ ParameterUtil ------------------------ protected final Map<String, String> data; protected final HashMap<String, String> defaultData; private ParameterTool(Map<String, String> data) { this.data = new HashMap<String, String>(data); this.defaultData = new HashMap<String, String>(); } // ------------------ Get data from the util ---------------- /** * Returns number of parameters in {@link ParameterTool}. */ public int getNumberOfParameters() { return data.size(); } /** * Returns the String value for the given key. * If the key does not exist it will return null. */ public String get(String key) { addToDefaults(key, null); return data.get(key); } /** * Returns the String value for the given key. * If the key does not exist it will throw a {@link RuntimeException}. */ public String getRequired(String key) { addToDefaults(key, null); String value = get(key); if (value == null) { throw new RuntimeException("No data for required key '" + key + "'"); } return value; } /** * Returns the String value for the given key. * If the key does not exist it will return the given default value. */ public String get(String key, String defaultValue) { addToDefaults(key, defaultValue); String value = get(key); if (value == null) { return defaultValue; } else { return value; } } /** * Check if value is set. */ public boolean has(String value) { addToDefaults(value, null); return data.containsKey(value); } // -------------- Integer /** * Returns the Integer value for the given key. * The method fails if the key does not exist or the value is not an Integer. */ public int getInt(String key) { addToDefaults(key, null); String value = getRequired(key); return Integer.parseInt(value); } /** * Returns the Integer value for the given key. If the key does not exists it will return the default value given. * The method fails if the value is not an Integer. */ public int getInt(String key, int defaultValue) { addToDefaults(key, Integer.toString(defaultValue)); String value = get(key); if (value == null) { return defaultValue; } return Integer.parseInt(value); } // -------------- LONG /** * Returns the Long value for the given key. * The method fails if the key does not exist. */ public long getLong(String key) { addToDefaults(key, null); String value = getRequired(key); return Long.parseLong(value); } /** * Returns the Long value for the given key. If the key does not exists it will return the default value given. * The method fails if the value is not a Long. */ public long getLong(String key, long defaultValue) { addToDefaults(key, Long.toString(defaultValue)); String value = get(key); if (value == null) { return defaultValue; } return Long.parseLong(value); } // -------------- FLOAT /** * Returns the Float value for the given key. * The method fails if the key does not exist. */ public float getFloat(String key) { addToDefaults(key, null); String value = getRequired(key); return Float.valueOf(value); } /** * Returns the Float value for the given key. If the key does not exists it will return the default value given. * The method fails if the value is not a Float. */ public float getFloat(String key, float defaultValue) { addToDefaults(key, Float.toString(defaultValue)); String value = get(key); if (value == null) { return defaultValue; } else { return Float.valueOf(value); } } // -------------- DOUBLE /** * Returns the Double value for the given key. * The method fails if the key does not exist. */ public double getDouble(String key) { addToDefaults(key, null); String value = getRequired(key); return Double.valueOf(value); } /** * Returns the Double value for the given key. If the key does not exists it will return the default value given. * The method fails if the value is not a Double. */ public double getDouble(String key, double defaultValue) { addToDefaults(key, Double.toString(defaultValue)); String value = get(key); if (value == null) { return defaultValue; } else { return Double.valueOf(value); } } // -------------- BOOLEAN /** * Returns the Boolean value for the given key. * The method fails if the key does not exist. */ public boolean getBoolean(String key) { addToDefaults(key, null); String value = getRequired(key); return Boolean.valueOf(value); } /** * Returns the Boolean value for the given key. If the key does not exists it will return the default value given. * The method returns whether the string of the value is "true" ignoring cases. */ public boolean getBoolean(String key, boolean defaultValue) { addToDefaults(key, Boolean.toString(defaultValue)); String value = get(key); if (value == null) { return defaultValue; } else { return Boolean.valueOf(value); } } // -------------- SHORT /** * Returns the Short value for the given key. * The method fails if the key does not exist. */ public short getShort(String key) { addToDefaults(key, null); String value = getRequired(key); return Short.valueOf(value); } /** * Returns the Short value for the given key. If the key does not exists it will return the default value given. * The method fails if the value is not a Short. */ public short getShort(String key, short defaultValue) { addToDefaults(key, Short.toString(defaultValue)); String value = get(key); if (value == null) { return defaultValue; } else { return Short.valueOf(value); } } // -------------- BYTE /** * Returns the Byte value for the given key. * The method fails if the key does not exist. */ public byte getByte(String key) { addToDefaults(key, null); String value = getRequired(key); return Byte.valueOf(value); } /** * Returns the Byte value for the given key. If the key does not exists it will return the default value given. * The method fails if the value is not a Byte. */ public byte getByte(String key, byte defaultValue) { addToDefaults(key, Byte.toString(defaultValue)); String value = get(key); if (value == null) { return defaultValue; } else { return Byte.valueOf(value); } } // --------------- Internals protected void addToDefaults(String key, String value) { String currentValue = defaultData.get(key); if (currentValue == null) { if (value == null) { value = DEFAULT_UNDEFINED; } defaultData.put(key, value); } else { // there is already an entry for this key. Check if the value is the undefined if (currentValue.equals(DEFAULT_UNDEFINED) && value != null) { // update key with better default value defaultData.put(key, value); } } } // ------------------------- Export to different targets ------------------------- /** * Returns a {@link Configuration} object from this {@link ParameterTool} * * @return A {@link Configuration} */ public Configuration getConfiguration() { Configuration conf = new Configuration(); for (Map.Entry<String, String> entry : data.entrySet()) { conf.setString(entry.getKey(), entry.getValue()); } return conf; } /** * Returns a {@link Properties} object from this {@link ParameterTool} * * @return A {@link Properties} */ public Properties getProperties() { Properties props = new Properties(); props.putAll(this.data); return props; } /** * Create a properties file with all the known parameters (call after the last get*() call). * Set the default value, if available. * * Use this method to create a properties file skeleton. * * @param pathToFile Location of the default properties file. */ public void createPropertiesFile(String pathToFile) throws IOException { createPropertiesFile(pathToFile, true); } /** * Create a properties file with all the known parameters (call after the last get*() call). * Set the default value, if overwrite is true. * * @param pathToFile Location of the default properties file. * @param overwrite Boolean flag indicating whether or not to overwrite the file * @throws IOException If overwrite is not allowed and the file exists */ public void createPropertiesFile(String pathToFile, boolean overwrite) throws IOException { File file = new File(pathToFile); if (file.exists()) { if (overwrite) { file.delete(); } else { throw new RuntimeException("File " + pathToFile + " exists and overwriting is not allowed"); } } Properties defaultProps = new Properties(); defaultProps.putAll(this.defaultData); try (final OutputStream out = new FileOutputStream(file)) { defaultProps.store(out, "Default file created by Flink's ParameterUtil.createPropertiesFile()"); } } @Override protected Object clone() throws CloneNotSupportedException { return new ParameterTool(this.data); } // ------------------------- Interaction with other ParameterUtils ------------------------- /** * Merges two {@link ParameterTool} * * @param other Other {@link ParameterTool} object * @return The Merged {@link ParameterTool} */ public ParameterTool mergeWith(ParameterTool other) { ParameterTool ret = new ParameterTool(this.data); ret.data.putAll(other.data); return ret; } // ------------------------- ExecutionConfig.UserConfig interface ------------------------- @Override public Map<String, String> toMap() { return data; } }