Java tutorial
/** * The contents of this file may be used under the terms of the Apache License, Version 2.0 * in which case, the provisions of the Apache License Version 2.0 are applicable instead of those above. * * Copyright 2014, Ecarf.io * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.ecarf.core.utils; import io.cloudex.framework.cloud.api.ApiUtils; import io.cloudex.framework.utils.FileUtils; import io.ecarf.core.term.TermPart; import io.ecarf.core.term.TermRoot; import io.ecarf.core.term.dictionary.TermDictionaryCommons; import io.ecarf.core.term.dictionary.TermDictionaryConcurrent; import io.ecarf.core.term.dictionary.TermDictionaryCore; import io.ecarf.core.term.dictionary.TermDictionaryGuava; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.BufferedWriter; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.ObjectInput; import java.io.ObjectInputStream; import java.io.ObjectOutput; import java.io.ObjectOutputStream; import java.io.OutputStream; import java.io.RandomAccessFile; import java.io.Serializable; import java.io.UnsupportedEncodingException; import java.lang.management.ManagementFactory; import java.lang.management.MemoryPoolMXBean; import java.lang.management.MemoryType; import java.lang.management.MemoryUsage; import java.lang.management.OperatingSystemMXBean; import java.lang.management.RuntimeMXBean; import java.net.URL; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.MappedByteBuffer; import java.nio.channels.Channels; import java.nio.channels.FileChannel; import java.nio.channels.FileChannel.MapMode; import java.nio.channels.ReadableByteChannel; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicInteger; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; import org.apache.commons.collections4.bidimap.DualHashBidiMap; import org.apache.commons.compress.compressors.gzip.GzipUtils; import org.apache.commons.lang.Validate; import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; import com.esotericsoftware.kryo.pool.KryoFactory; import com.esotericsoftware.kryo.pool.KryoPool; import com.esotericsoftware.kryo.serializers.MapSerializer; import com.google.common.collect.HashBiMap; import com.google.gson.Gson; import com.google.gson.JsonIOException; import com.google.gson.JsonSyntaxException; import com.google.gson.reflect.TypeToken; import com.google.gson.stream.JsonReader; /** * @author Omer Dawelbeit (omerio) * */ public class Utils { private final static Log log = LogFactory.getLog(Utils.class); public static final String PATH_SEPARATOR = File.separator; public static final String TEMP_FOLDER = System.getProperty("java.io.tmpdir") + PATH_SEPARATOR; public static final byte[] SEPARATOR = System.getProperty("line.separator").getBytes(); public static final int BUFFER_SIZE = 8192; public static Gson GSON = new Gson(); public static final KryoPool POOL; static { KryoFactory factory = new KryoFactory() { public Kryo create() { Kryo kryo = new Kryo(); // configure kryo instance, customize settings MapSerializer mapSerializer = new MapSerializer(); mapSerializer.setKeysCanBeNull(false); /*CollectionSerializer colSerializer = new CollectionSerializer(); colSerializer.setImmutable(true); colSerializer.setElementsCanBeNull(false);*/ kryo.register(TermDictionaryCore.class); kryo.register(HashSet.class); kryo.register(HashMap.class, mapSerializer); //kryo.register(HashBiMap.class, serializer); kryo.register(HashBiMap.class, new MapSerializer() { @SuppressWarnings("rawtypes") public Map create(Kryo kryo, Input input, Class<Map> type) { return HashBiMap.create(); } }); kryo.register(TermRoot.class); kryo.register(TermPart.class); kryo.register(TermDictionaryGuava.class); kryo.register(TermDictionaryCommons.class); kryo.register(TermDictionaryConcurrent.class); kryo.register(ConcurrentHashMap.class); kryo.register(AtomicInteger.class); kryo.register(DualHashBidiMap.class); kryo.setRegistrationRequired(true); return kryo; } }; // Build pool with SoftReferences enabled (optional) POOL = new KryoPool.Builder(factory).softReferences().build(); } /** * Split a string using indexOf and subsString, this is proved to be * faster than String.split and StringUtils.split * @param strToSplit * @param delimiter * @return */ public static List<String> split(String strToSplit, char delimiter) { List<String> parts = new ArrayList<>(); int foundPosition; int startIndex = 0; String part; while ((foundPosition = strToSplit.indexOf(delimiter, startIndex)) > -1) { part = strToSplit.substring(startIndex, foundPosition); if (part.length() > 0) { parts.add(part); } startIndex = foundPosition + 1; } if (startIndex < strToSplit.length()) { parts.add(strToSplit.substring(startIndex)); } return parts; } /** * Split a string using indexOf and subsString, this is proved to be * faster than String.split and StringUtils.split * @param strToSplit * @param delimiter * @return */ public static Set<String> splitNoDuplicates(String strToSplit, char delimiter) { Set<String> parts = new HashSet<>(); int foundPosition; int startIndex = 0; String part; while ((foundPosition = strToSplit.indexOf(delimiter, startIndex)) > -1) { part = strToSplit.substring(startIndex, foundPosition); if (part.length() > 0) { parts.add(part); } startIndex = foundPosition + 1; } if (startIndex < strToSplit.length()) { parts.add(strToSplit.substring(startIndex)); } return parts; } /** * Get the likely uncompressed size of a Gziped file * @param filename * @return * @throws IOException * @see http://stackoverflow.com/questions/27825927/get-gzipped-file-attributes-like-gzip-l-basically-compression-ratio * @throws FileNotFoundException */ public static long getUncompressedFileSize(String filename) throws FileNotFoundException, IOException { File f = new File(filename); try (RandomAccessFile ra = new RandomAccessFile(f, "r"); FileChannel channel = ra.getChannel()) { MappedByteBuffer fileBuffer = channel.map(MapMode.READ_ONLY, f.length() - 4, 4); fileBuffer.load(); ByteBuffer buf = ByteBuffer.allocate(4); buf.order(ByteOrder.LITTLE_ENDIAN); buf.put(fileBuffer); buf.flip(); //will print the uncompressed size //getInt() reads the 4 bytes as a int // if the file is between 2GB and 4GB // then this will return a negative value //and you'll have to do your own converting to an unsigned int int size = buf.getInt(); if (size < 0) { return FileUtils.ONE_GB + size; } else { return size; } } } /** * Copy a url to a local file * @param url * @param file * @throws IOException */ public static void downloadFile(String url, String file) throws IOException { log.info("Downloading file from: " + url); URL webfile = new URL(url); try (ReadableByteChannel rbc = Channels.newChannel(webfile.openStream()); FileOutputStream fos = new FileOutputStream(file)) { fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); } } /** * Wait until two values become equal * @param value1 * @param value2 */ public static void waitForEquality(Object value1, Object value2, int seconds) { while (!value1.equals(value2)) { ApiUtils.block(seconds); } } /** * Merge two count maps of type <K, Integer> * @param <K> * @param base * @param other */ public static <K> void mergeCountMaps(Map<K, Integer> base, Map<K, Integer> other) { for (Entry<K, Integer> entry : other.entrySet()) { K key = entry.getKey(); Integer value = entry.getValue(); if (base.get(key) == null) { base.put(key, value); } else { base.put(key, base.get(key) + value); } } } public static void main(String[] args) throws JsonIOException, JsonSyntaxException, FileNotFoundException { Set<String> schemaTerms = Utils.GSON.fromJson( new JsonReader(new FileReader(Utils.TEMP_FOLDER + "test.json")), new TypeToken<Set<String>>() { }.getType()); System.out.println(schemaTerms); } /** * Create a fixed size thread pool * @param numOfThreads, the number of threads, if null defaults to the number of * processors * @return */ public static ExecutorService createFixedThreadPool(Integer numOfThreads) { if (numOfThreads == null) { numOfThreads = Runtime.getRuntime().availableProcessors(); } log.info("Creating fixed thread pool of size: " + numOfThreads); ExecutorService executor = Executors.newFixedThreadPool(numOfThreads); return executor; } public static int getStringSize(String text) { int size = 0; if (StringUtils.isNotBlank(text)) { try { size = text.getBytes(Constants.UTF8).length; } catch (UnsupportedEncodingException e) { log.warn("Failed to encode text as UTF8: " + text, e); } } return size; } /** * * @param filename * @param object * @throws IOException */ public static void objectToJsonFile(String filename, Object object) throws IOException { try (BufferedWriter writer = new BufferedWriter(new FileWriter(filename), Constants.GZIP_BUF_SIZE)) { GSON.toJson(object, writer); } } /** * * @param filename * @param object * @param compress * @throws IOException */ public static void objectToFile(String filename, Object object, boolean compress) throws IOException { objectToFile(filename, object, compress, true); } /** * Serialize an object to a file * @param filename * @param object * @throws IOException */ public static void objectToFile(String filename, Object object, boolean compress, boolean java) throws IOException { Validate.isTrue(object instanceof Serializable, "object must implement Serializable"); log.info("Serializing object of class: " + object.getClass() + " to file: " + filename + ", with compress = " + compress); OutputStream stream = new FileOutputStream(filename); if (compress) { stream = new GZIPOutputStream(stream, Constants.GZIP_BUF_SIZE); } //else { stream = new BufferedOutputStream(stream, Constants.GZIP_BUF_SIZE); //} if (java) { try (ObjectOutput oos = new ObjectOutputStream(stream);) { oos.writeObject(object); } } else { Kryo kryo = POOL.borrow(); // do s.th. with kryo here, and afterwards release it try (Output output = new Output(stream, Constants.GZIP_BUF_SIZE)) { kryo.writeObject(output, object); } POOL.release(kryo); } } /** * Use java serialization into file * @param filename * @param classOfT * @param compressed * @return * @throws ClassNotFoundException * @throws FileNotFoundException * @throws IOException */ public static <T> T objectFromFile(String filename, Class<T> classOfT, boolean compressed) throws ClassNotFoundException, FileNotFoundException, IOException { return objectFromFile(filename, classOfT, compressed, true); } /** * Read a serialized object from file * @param filename * @param classOfT * @return * @throws FileNotFoundEx * @throws ClassNotFoundException ception * @throws IOException */ @SuppressWarnings("unchecked") public static <T> T objectFromFile(String filename, Class<T> classOfT, boolean compressed, boolean java) throws ClassNotFoundException, FileNotFoundException, IOException { log.info("Reading object of class: " + classOfT + " from file: " + filename + ", with compressed = " + compressed); T object = null; InputStream stream = new FileInputStream(filename); if (compressed) { stream = new GZIPInputStream(stream, Constants.GZIP_BUF_SIZE); } //else { stream = new BufferedInputStream(stream, Constants.GZIP_BUF_SIZE); //} if (java) { try (ObjectInput ois = new ObjectInputStream(stream);) { object = (T) ois.readObject(); } } else { Kryo kryo = POOL.borrow(); // do s.th. with kryo here, and afterwards release it try (Input input = new Input(stream, Constants.GZIP_BUF_SIZE)) { object = kryo.readObject(input, classOfT); } POOL.release(kryo); } // SomeClass someObject = kryo.readObject(input, SomeClass.class); return object; } /** * Compress a file * @param inFile * @param outFile * @throws FileNotFoundException * @throws IOException */ public static String compressFile(String inFile) throws FileNotFoundException, IOException { byte[] buffer = new byte[BUFFER_SIZE]; String outFile = GzipUtils.getCompressedFilename(inFile); log.info("Compressing file: " + inFile + ", to file: " + outFile); try (GZIPOutputStream gzos = new GZIPOutputStream(new FileOutputStream(outFile), Constants.GZIP_BUF_SIZE); FileInputStream in = new FileInputStream(inFile);) { int len; while ((len = in.read(buffer)) > 0) { gzos.write(buffer, 0, len); } gzos.finish(); } return outFile; } /** * (Gzip) Uncompress a compressed file * @param filename * @return * @throws IOException */ public static String unCompressFile(String filename) throws IOException { byte[] buffer = new byte[BUFFER_SIZE]; String outFile = GzipUtils.getUncompressedFilename(filename); log.info("Uncompressing file: " + filename + ", to file: " + outFile); try (GZIPInputStream gzis = new GZIPInputStream(new FileInputStream(filename), Constants.GZIP_BUF_SIZE); BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(outFile), Constants.GZIP_BUF_SIZE);) { int len; while ((len = gzis.read(buffer)) > 0) { out.write(buffer, 0, len); } } /* FileInputStream fin = new FileInputStream(filename); BufferedInputStream in = new BufferedInputStream(fin); try(FileOutputStream out = new FileOutputStream(outFile); GzipCompressorInputStream gzIn = new GzipCompressorInputStream(in)) { final byte[] buffer = new byte[BUFFER]; int n = 0; while (-1 != (n = gzIn.read(buffer))) { out.write(buffer, 0, n); } }*/ return outFile; } /** * Log the CPU stats */ public static void logCpuStats() { OperatingSystemMXBean operatingSystemMXBean = (OperatingSystemMXBean) ManagementFactory .getOperatingSystemMXBean(); RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean(); int availableProcessors = operatingSystemMXBean.getAvailableProcessors(); long upTime = runtimeMXBean.getUptime(); double loadAverage = operatingSystemMXBean.getSystemLoadAverage(); log.info("Available processors: " + availableProcessors); log.info("JVM upTime (ms): " + upTime); log.info("CPU load average: " + loadAverage); } /** * Log memory usage * @return */ public static long getMemoryUsage() { List<MemoryPoolMXBean> memoryPools = new ArrayList<MemoryPoolMXBean>( ManagementFactory.getMemoryPoolMXBeans()); long usedHeapMemoryAfterLastGC = 0; for (MemoryPoolMXBean memoryPool : memoryPools) { if (memoryPool.getType().equals(MemoryType.HEAP)) { MemoryUsage poolCollectionMemoryUsage = memoryPool.getCollectionUsage(); usedHeapMemoryAfterLastGC += poolCollectionMemoryUsage.getUsed(); } } return usedHeapMemoryAfterLastGC; } /** * Get the memory usage in GB * @return */ public static double getMemoryUsageInGB() { double gb = ((double) getMemoryUsage() / FileUtils.ONE_GB); return gb; } /** * Log the memory usage in Gigabytes */ public static void logMemoryUsageInGB() { double gb = getMemoryUsageInGB(); log.info("JVM Memory usage: " + gb + "GB"); } // /* // * Determines if a byte array is compressed. The java.util.zip GZip // * Implementation does not expose the GZip header so it is difficult to determine // * if a string is compressed. // * // * @param bytes an array of bytes // * @return true if the array is compressed or false otherwise // * @throws java.io.IOException if the byte array couldn't be read // */ // public static boolean isGziped(byte[] bytes) throws IOException { // boolean compressed = false; // if ((bytes == null) || (bytes.length < 2)) { // // compressed = ((bytes[0] == (byte) (GZIPInputStream.GZIP_MAGIC)) && // (bytes[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8))); // } // // return compressed; // } // // /** // * Checks if an input stream is gzipped. // * // * @param in // * @return // */ // public static boolean isGZipped(InputStream in) { // if (!in.markSupported()) { // in = new BufferedInputStream(in); // } // in.mark(2); // int magic = 0; // try { // magic = in.read() & 0xff | ((in.read() << 8) & 0xff00); // in.reset(); // } catch (IOException e) { // log.log(Level.SEVERE, "Failed to read file", e); // return false; // } // return magic == GZIPInputStream.GZIP_MAGIC; // } // // /** // * Checks if a file is gzipped. // * // * @param f // * @return // */ // public static boolean isGZipped(File f) { // int magic = 0; // try { // RandomAccessFile raf = new RandomAccessFile(f, "r"); // magic = raf.read() & 0xff | ((raf.read() << 8) & 0xff00); // raf.close(); // } catch (Throwable e) { // log.log(Level.SEVERE, "Failed to read file", e); // } // return magic == GZIPInputStream.GZIP_MAGIC; // } /** * Converts the query results retrieved from the datastore to json parsable by javascript * into a DataTable object for use with a motion chart. */ /* private void writeResultsToMotionChartJson(JsonWriter jsonWriter, Iterable<Entity> results) throws IOException { jsonWriter.name("data").beginObject(); // Write the header. jsonWriter.name("cols").beginArray(); for (int i = 0; i < properties.length; i++) { jsonWriter.beginObject() .name("id").value(properties[i]) .name("label").value(labels[i]) .name("type").value(types[i]) .endObject(); } jsonWriter.endArray(); // Write the data. jsonWriter.name("rows").beginArray(); for (Entity entity : results) { jsonWriter.beginObject().name("c").beginArray(); for (int i = 0; i < properties.length; i++) { String value = ""; if (entity.getProperty(properties[i]) != null) { value = String.valueOf(entity.getProperty(properties[i])); } jsonWriter.beginObject().name("v").value(value).endObject(); } jsonWriter.endArray().endObject(); } jsonWriter.endArray(); jsonWriter.endObject(); }*/ }