Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * <p/> * http://www.apache.org/licenses/LICENSE-2.0 * <p/> * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.alibaba.jstorm.yarn.utils; import org.apache.commons.exec.*; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.thrift.TFieldIdEnum; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.management.ObjectName; import java.io.*; import java.lang.management.ManagementFactory; import java.lang.management.MemoryMXBean; import java.lang.management.MemoryUsage; import java.math.BigInteger; import java.nio.ByteBuffer; import java.util.*; import java.util.Map.Entry; import java.util.concurrent.TimeUnit; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; /** * JStorm utility * * @author yannian/Longda/Xin.Zhou/Xin.Li */ public class JStormUtils { private static final Logger LOG = LoggerFactory.getLogger(JStormUtils.class); public static long SIZE_1_K = 1024; public static long SIZE_1_M = SIZE_1_K * 1024; public static long SIZE_1_G = SIZE_1_M * 1024; public static long SIZE_1_T = SIZE_1_G * 1024; public static long SIZE_1_P = SIZE_1_T * 1024; public static final int MIN_1 = 60; public static final int MIN_30 = MIN_1 * 30; public static final int HOUR_1 = MIN_30 * 2; public static final int DAY_1 = HOUR_1 * 24; public static String getErrorInfo(String baseInfo, Exception e) { try { StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); e.printStackTrace(pw); return baseInfo + "\r\n" + sw.toString() + "\r\n"; } catch (Exception e2) { return baseInfo; } } public static String getErrorInfo(Throwable error) { try { StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); error.printStackTrace(pw); return sw.toString(); } catch (Exception e1) { return ""; } } /** * filter the map * * @param filter * @param all * @return */ public static <K, V> Map<K, V> select_keys_pred(Set<K> filter, Map<K, V> all) { Map<K, V> filterMap = new HashMap<K, V>(); for (Entry<K, V> entry : all.entrySet()) { if (!filter.contains(entry.getKey())) { filterMap.put(entry.getKey(), entry.getValue()); } } return filterMap; } public static byte[] barr(byte v) { byte[] byteArray = new byte[1]; byteArray[0] = v; return byteArray; } public static byte[] barr(Short v) { byte[] byteArray = new byte[Short.SIZE / 8]; for (int i = 0; i < byteArray.length; i++) { int off = (byteArray.length - 1 - i) * 8; byteArray[i] = (byte) ((v >> off) & 0xFF); } return byteArray; } public static byte[] barr(Integer v) { byte[] byteArray = new byte[Integer.SIZE / 8]; for (int i = 0; i < byteArray.length; i++) { int off = (byteArray.length - 1 - i) * 8; byteArray[i] = (byte) ((v >> off) & 0xFF); } return byteArray; } // for test public static int byteToInt2(byte[] b) { int iOutcome = 0; byte bLoop; for (int i = 0; i < 4; i++) { bLoop = b[i]; int off = (b.length - 1 - i) * 8; iOutcome += (bLoop & 0xFF) << off; } return iOutcome; } /** * LocalMode variable isn't clean, it make the JStormUtils ugly */ public static boolean localMode = false; public static boolean isLocalMode() { return localMode; } public static void setLocalMode(boolean localMode) { JStormUtils.localMode = localMode; } public static void haltProcess(int val) { Runtime.getRuntime().halt(val); } public static void halt_process(int val, String msg) { LOG.info("Halting process: " + msg); try { Thread.sleep(1000); } catch (InterruptedException e) { } if (localMode && val == 0) { // throw new RuntimeException(msg); } else { haltProcess(val); } } /** * "{:a 1 :b 1 :c 2} -> {1 [:a :b] 2 :c}" * * @param map * @return */ public static <K, V> HashMap<V, List<K>> reverse_map(Map<K, V> map) { HashMap<V, List<K>> rtn = new HashMap<V, List<K>>(); if (map == null) { return rtn; } for (Entry<K, V> entry : map.entrySet()) { K key = entry.getKey(); V val = entry.getValue(); List<K> list = rtn.get(val); if (list == null) { list = new ArrayList<K>(); rtn.put(entry.getValue(), list); } list.add(key); } return rtn; } /** * Gets the pid of this JVM, because Java doesn't provide a real way to do this. * * @return */ public static String process_pid() { String name = ManagementFactory.getRuntimeMXBean().getName(); String[] split = name.split("@"); if (split.length != 2) { throw new RuntimeException("Got unexpected process name: " + name); } return split[0]; } /** * All exe command will go launchProcess * * @param command * @throws ExecuteException * @throws IOException */ public static void exec_command(String command) throws ExecuteException, IOException { launchProcess(command, new HashMap<String, String>(), false); } /** * Extra dir from the jar to destdir * * @param jarpath * @param dir * @param destdir */ public static void extractDirFromJar(String jarpath, String dir, String destdir) { ZipFile zipFile = null; try { zipFile = new ZipFile(jarpath); Enumeration<? extends ZipEntry> entries = zipFile.entries(); while (entries != null && entries.hasMoreElements()) { ZipEntry ze = entries.nextElement(); if (!ze.isDirectory() && ze.getName().startsWith(dir)) { InputStream in = zipFile.getInputStream(ze); try { File file = new File(destdir, ze.getName()); if (!file.getParentFile().mkdirs()) { if (!file.getParentFile().isDirectory()) { throw new IOException("Mkdirs failed to create " + file.getParentFile().toString()); } } OutputStream out = new FileOutputStream(file); try { byte[] buffer = new byte[8192]; int i; while ((i = in.read(buffer)) != -1) { out.write(buffer, 0, i); } } finally { out.close(); } } finally { if (in != null) in.close(); } } } } catch (Exception e) { LOG.warn("No " + dir + " from " + jarpath + "!\n" + e.getMessage()); } finally { if (zipFile != null) try { zipFile.close(); } catch (Exception e) { LOG.warn(e.getMessage()); } } } public static void ensure_process_killed(Integer pid) { // in this function, just kill the process 5 times // make sure the process be killed definitely for (int i = 0; i < 5; i++) { try { exec_command("kill -9 " + pid); LOG.info("kill -9 process " + pid); sleepMs(100); } catch (ExecuteException e) { LOG.info("Error when trying to kill " + pid + ". Process has been killed"); } catch (Exception e) { LOG.info("Error when trying to kill " + pid + ".Exception ", e); } } } public static void process_killed(Integer pid) { try { exec_command("kill " + pid); LOG.info("kill process " + pid); } catch (ExecuteException e) { LOG.info("Error when trying to kill " + pid + ". Process has been killed. "); } catch (Exception e) { LOG.info("Error when trying to kill " + pid + ".Exception ", e); } } public static void kill(Integer pid) { process_killed(pid); sleepMs(5 * 1000); ensure_process_killed(pid); } public static void kill_signal(Integer pid, String signal) { String cmd = "kill " + signal + " " + pid; try { exec_command(cmd); LOG.info(cmd); } catch (ExecuteException e) { LOG.info("Error when run " + cmd + ". Process has been killed. "); } catch (Exception e) { LOG.info("Error when run " + cmd + ". Exception ", e); } } /** * This function is only for linux * * @param pid * @return */ public static boolean isProcDead(String pid) { if (OSInfo.isLinux() == false) { return false; } String path = "/proc/" + pid; File file = new File(path); if (file.exists() == false) { LOG.info("Process " + pid + " is dead"); return true; } return false; } /** * If it is backend, please set resultHandler, such as DefaultExecuteResultHandler If it is frontend, ByteArrayOutputStream.toString get the result * <p/> * This function don't care whether the command is successfully or not * * @param command * @param environment * @param workDir * @param resultHandler * @return * @throws IOException */ @Deprecated public static ByteArrayOutputStream launchProcess(String command, final Map environment, final String workDir, ExecuteResultHandler resultHandler) throws IOException { String[] cmdlist = command.split(" "); CommandLine cmd = new CommandLine(cmdlist[0]); for (String cmdItem : cmdlist) { if (StringUtils.isBlank(cmdItem) == false) { cmd.addArgument(cmdItem); } } DefaultExecutor executor = new DefaultExecutor(); executor.setExitValue(0); if (StringUtils.isBlank(workDir) == false) { executor.setWorkingDirectory(new File(workDir)); } ByteArrayOutputStream out = new ByteArrayOutputStream(); PumpStreamHandler streamHandler = new PumpStreamHandler(out, out); if (streamHandler != null) { executor.setStreamHandler(streamHandler); } try { if (resultHandler == null) { executor.execute(cmd, environment); } else { executor.execute(cmd, environment, resultHandler); } } catch (ExecuteException e) { // @@@@ // failed to run command } return out; } protected static Process launchProcess(final List<String> cmdlist, final Map<String, String> environment) throws IOException { ProcessBuilder builder = new ProcessBuilder(cmdlist); builder.redirectErrorStream(true); Map<String, String> process_evn = builder.environment(); for (Entry<String, String> entry : environment.entrySet()) { process_evn.put(entry.getKey(), entry.getValue()); } return builder.start(); } public static String getOutput(InputStream input) { BufferedReader in = new BufferedReader(new InputStreamReader(input)); StringBuilder sb = new StringBuilder(); String line; try { while ((line = in.readLine()) != null) { sb.append(line).append("\n"); } } catch (IOException e) { e.printStackTrace(); } finally { try { in.close(); } catch (IOException e) { e.printStackTrace(); } } return sb.toString(); } public static String launchProcess(final String command, final List<String> cmdlist, final Map<String, String> environment, boolean backend) throws IOException { if (backend) { new Thread(new Runnable() { @Override public void run() { List<String> cmdWrapper = new ArrayList<String>(); cmdWrapper.add("nohup"); cmdWrapper.addAll(cmdlist); cmdWrapper.add("&"); try { launchProcess(cmdWrapper, environment); } catch (IOException e) { LOG.error("Failed to run nohup " + command + " &," + e.getCause(), e); } } }).start(); return null; } else { try { Process process = launchProcess(cmdlist, environment); StringBuilder sb = new StringBuilder(); String output = JStormUtils.getOutput(process.getInputStream()); String errorOutput = JStormUtils.getOutput(process.getErrorStream()); sb.append(output); sb.append("\n"); sb.append(errorOutput); int ret = process.waitFor(); if (ret != 0) { LOG.warn(command + " is terminated abnormally. ret={}, str={}", ret, sb.toString()); } return sb.toString(); } catch (Throwable e) { LOG.error("Failed to run " + command + ", " + e.getCause(), e); } return ""; } } /** * Attention * * All launchProcess should go this function * * it should use DefaultExecutor to start a process, * but some little problem have been found, * such as exitCode/output string so still use the old method to start process * * @param command * @param environment * @param backend * @return outputString * @throws IOException */ public static String launchProcess(final String command, final Map<String, String> environment, boolean backend) throws IOException { String[] cmds = command.split(" "); ArrayList<String> cmdList = new ArrayList<String>(); for (String tok : cmds) { if (!StringUtils.isBlank(tok)) { cmdList.add(tok); } } return launchProcess(command, cmdList, environment, backend); } public static String current_classpath() { return System.getProperty("java.class.path"); } public static <V> HashMap<V, Integer> multi_set(List<V> list) { HashMap<V, Integer> rtn = new HashMap<V, Integer>(); for (V v : list) { int cnt = 1; if (rtn.containsKey(v)) { cnt += rtn.get(v); } rtn.put(v, cnt); } return rtn; } /** * if the list exist repeat string, return the repeated string * <p/> * this function will be used to check wheter bolt or spout exist same id */ public static List<String> getRepeat(List<String> list) { List<String> rtn = new ArrayList<String>(); Set<String> idSet = new HashSet<String>(); for (String id : list) { if (idSet.contains(id)) { rtn.add(id); } else { idSet.add(id); } } return rtn; } /** * balance all T * * @param <T> * @param splitup * @return */ public static <T> List<T> interleave_all(List<List<T>> splitup) { ArrayList<T> rtn = new ArrayList<T>(); int maxLength = 0; for (List<T> e : splitup) { int len = e.size(); if (maxLength < len) { maxLength = len; } } for (int i = 0; i < maxLength; i++) { for (List<T> e : splitup) { if (e.size() > i) { rtn.add(e.get(i)); } } } return rtn; } public static long bit_xor_vals(Object... vals) { long rtn = 0l; for (Object n : vals) { rtn = bit_xor(rtn, n); } return rtn; } public static <T> long bit_xor_vals(List<T> vals) { long rtn = 0l; for (T n : vals) { rtn = bit_xor(rtn, n); } return rtn; } public static <T> long bit_xor_vals_sets(Set<T> vals) { long rtn = 0l; for (T n : vals) { rtn = bit_xor(rtn, n); } return rtn; } public static long bit_xor(Object a, Object b) { long rtn; if (a instanceof Long && b instanceof Long) { rtn = ((Long) a) ^ ((Long) b); return rtn; } else if (b instanceof Set) { long bs = bit_xor_vals_sets((Set) b); return bit_xor(a, bs); } else if (a instanceof Set) { long as = bit_xor_vals_sets((Set) a); return bit_xor(as, b); } else { long ai = Long.parseLong(String.valueOf(a)); long bi = Long.parseLong(String.valueOf(b)); rtn = ai ^ bi; return rtn; } } public static <V> List<V> mk_list(V... args) { ArrayList<V> rtn = new ArrayList<V>(); for (V o : args) { rtn.add(o); } return rtn; } public static <V> List<V> mk_list(Set<V> args) { ArrayList<V> rtn = new ArrayList<V>(); if (args != null) { for (V o : args) { rtn.add(o); } } return rtn; } public static <V> List<V> mk_list(Collection<V> args) { ArrayList<V> rtn = new ArrayList<V>(); if (args != null) { for (V o : args) { rtn.add(o); } } return rtn; } public static <V> V[] mk_arr(V... args) { return args; } public static Long parseLong(Object o) { if (o == null) { return null; } if (o instanceof String) { return Long.valueOf(String.valueOf(o)); } else if (o instanceof Integer) { Integer value = (Integer) o; return Long.valueOf(value); } else if (o instanceof Long) { return (Long) o; } else { throw new RuntimeException("Invalid value " + o.getClass().getName() + " " + o); } } public static Double parseDouble(Object o) { if (o == null) { return null; } if (o instanceof String) { return Double.valueOf(String.valueOf(o)); } else if (o instanceof Integer) { Number value = (Integer) o; return value.doubleValue(); } else if (o instanceof Long) { Number value = (Long) o; return value.doubleValue(); } else if (o instanceof Double) { return (Double) o; } else { throw new RuntimeException("Invalid value " + o.getClass().getName() + " " + o); } } public static Double parseDouble(Object o, double defaultValue) { if (o == null) { return defaultValue; } try { return parseDouble(o); } catch (Exception ignored) { return defaultValue; } } public static Long parseLong(Object o, long defaultValue) { if (o == null) { return defaultValue; } if (o instanceof String) { return Long.valueOf(String.valueOf(o)); } else if (o instanceof Integer) { Integer value = (Integer) o; return Long.valueOf((Integer) value); } else if (o instanceof Long) { return (Long) o; } else { return defaultValue; } } public static Integer parseInt(Object o) { if (o == null) { return null; } if (o instanceof String) { return Integer.parseInt(String.valueOf(o)); } else if (o instanceof Long) { long value = (Long) o; return (int) value; } else if (o instanceof Integer) { return (Integer) o; } else { throw new RuntimeException("Invalid value " + o.getClass().getName() + " " + o); } } public static Integer parseInt(Object o, int defaultValue) { if (o == null) { return defaultValue; } if (o instanceof String) { return Integer.parseInt(String.valueOf(o)); } else if (o instanceof Long) { long value = (Long) o; return (int) value; } else if (o instanceof Integer) { return (Integer) o; } else { return defaultValue; } } public static Boolean parseBoolean(Object o) { if (o == null) { return null; } if (o instanceof String) { return Boolean.valueOf((String) o); } else if (o instanceof Boolean) { return (Boolean) o; } else { throw new RuntimeException("Invalid value " + o.getClass().getName() + " " + o); } } public static boolean parseBoolean(Object o, boolean defaultValue) { if (o == null) { return defaultValue; } if (o instanceof String) { return Boolean.valueOf((String) o); } else if (o instanceof Boolean) { return (Boolean) o; } else { return defaultValue; } } public static <V> Set<V> listToSet(List<V> list) { if (list == null) { return null; } Set<V> set = new HashSet<V>(); set.addAll(list); return set; } /** * Check whether the zipfile contain the resources * * @param zipfile * @param resources * @return */ public static boolean zipContainsDir(String zipfile, String resources) { Enumeration<? extends ZipEntry> entries = null; try { entries = (new ZipFile(zipfile)).entries(); while (entries != null && entries.hasMoreElements()) { ZipEntry ze = entries.nextElement(); String name = ze.getName(); if (name.startsWith(resources + "/")) { return true; } } } catch (IOException e) { // TODO Auto-generated catch block // e.printStackTrace(); LOG.error(e + "zipContainsDir error"); } return false; } public static Object add(Object oldValue, Object newValue) { if (oldValue == null) { return newValue; } else if (newValue == null) { return oldValue; } if (oldValue instanceof Long) { return (Long) oldValue + (Long) newValue; } else if (oldValue instanceof Double) { return (Double) oldValue + (Double) newValue; } else if (oldValue instanceof Integer) { return (Integer) oldValue + (Integer) newValue; } else if (oldValue instanceof Float) { return (Float) oldValue + (Float) newValue; } else if (oldValue instanceof Short) { return (Short) oldValue + (Short) newValue; } else if (oldValue instanceof BigInteger) { return ((BigInteger) oldValue).add((BigInteger) newValue); } else if (oldValue instanceof Number) { return ((Number) oldValue).doubleValue() + ((Number) newValue).doubleValue(); } else { return null; } } public static Object mergeList(List<Object> list) { Object ret = null; for (Object value : list) { ret = add(ret, value); } return ret; } public static List<Object> mergeList(List<Object> result, Object add) { if (add instanceof Collection) { for (Object o : (Collection) add) { result.add(o); } } else if (add instanceof Set) { for (Object o : (Collection) add) { result.add(o); } } else { result.add(add); } return result; } public static List<Object> distinctList(List<Object> input) { List<Object> retList = new ArrayList<Object>(); for (Object object : input) { if (retList.contains(object)) { continue; } else { retList.add(object); } } return retList; } public static <K, V> Map<K, V> mergeMapList(List<Map<K, V>> list) { Map<K, V> ret = new HashMap<K, V>(); for (Map<K, V> listEntry : list) { if (listEntry == null) { continue; } for (Entry<K, V> mapEntry : listEntry.entrySet()) { K key = mapEntry.getKey(); V value = mapEntry.getValue(); V retValue = (V) add(ret.get(key), value); ret.put(key, retValue); } } return ret; } public static String formatSimpleDouble(Double value) { try { java.text.DecimalFormat form = new java.text.DecimalFormat("##0.000"); String s = form.format(value); return s; } catch (Exception e) { return "0.000"; } } public static double formatDoubleDecPoint2(Double value) { try { java.text.DecimalFormat form = new java.text.DecimalFormat("##.00"); String s = form.format(value); return Double.valueOf(s); } catch (Exception e) { return 0.0; } } public static double formatDoubleDecPoint4(Double value) { try { java.text.DecimalFormat form = new java.text.DecimalFormat("###.0000"); String s = form.format(value); return Double.valueOf(s); } catch (Exception e) { return 0.0; } } public static Double convertToDouble(Object value) { Double ret; if (value == null) { ret = null; } else { if (value instanceof Integer) { ret = ((Integer) value).doubleValue(); } else if (value instanceof Long) { ret = ((Long) value).doubleValue(); } else if (value instanceof Float) { ret = ((Float) value).doubleValue(); } else if (value instanceof Double) { ret = (Double) value; } else { ret = null; } } return ret; } public static String formatValue(Object value) { if (value == null) { return "0"; } if (value instanceof Long) { return String.valueOf((Long) value); } else if (value instanceof Double) { return formatSimpleDouble((Double) value); } else { return String.valueOf(value); } } public static void sleepMs(long ms) { try { Thread.sleep(ms); } catch (InterruptedException e) { } } public static void sleepNs(int ns) { try { Thread.sleep(0, ns); } catch (InterruptedException e) { } } public static String HEXES = "0123456789ABCDEF"; public static String toPrintableString(byte[] buf) { if (buf == null) { return null; } StringBuilder sb = new StringBuilder(); int index = 0; for (byte b : buf) { if (index % 10 == 0) { sb.append("\n"); } index++; sb.append(HEXES.charAt((b & 0xF0) >> 4)); sb.append(HEXES.charAt((b & 0x0F))); sb.append(" "); } return sb.toString(); } /** * @return * @@@ Todo */ public static Long getPhysicMemorySize() { Object object; try { object = ManagementFactory.getPlatformMBeanServer().getAttribute( new ObjectName("java.lang", "type", "OperatingSystem"), "TotalPhysicalMemorySize"); } catch (Exception e) { LOG.warn("Failed to get system physical memory size,", e); return null; } Long ret = (Long) object; return ret; } public static String genLogName(String topology, Integer port) { return topology + "-worker-" + port + ".log"; } public static byte[] longToBytes(long x) { ByteBuffer buffer = ByteBuffer.allocate(Long.SIZE); buffer.putLong(x); return buffer.array(); } public static long bytesToLong(byte[] bytes) { ByteBuffer buffer = ByteBuffer.allocate(Long.SIZE); buffer.put(bytes); buffer.flip();// need flip return buffer.getLong(); } public static long halfValueOfSum(long v1, long v2, boolean increment) { long ret = (v1 + v2) / 2; if (increment) { ret += (v1 + v2) % 2; } return ret; } public static String join(String delim, List<String> strings) { StringBuilder builder = new StringBuilder(); if (strings != null) { for (String str : strings) { if (builder.length() > 0) { builder.append(delim).append(" "); } builder.append(str); } } return builder.toString(); } }