Java tutorial
/******************************************************************************* * Copyright (c) 2000, 2017 IBM Corporation and others. * * This program and the accompanying materials * are made available under the terms of the Eclipse Public License 2.0 * which accompanies this distribution, and is available at * https://www.eclipse.org/legal/epl-2.0/ * * SPDX-License-Identifier: EPL-2.0 * * Contributors: * IBM Corporation - initial API and implementation *******************************************************************************/ package org.eclipse.swt.internal; import java.io.*; import java.nio.*; import java.nio.charset.*; import org.eclipse.swt.internal.gtk.*; /** * About this class: * ################# * This class implements the conversions between unicode characters * and the platform supported representation for characters. * * Note that, unicode characters which can not be found in the platform * encoding will be converted to an arbitrary platform specific character. * * This class is tested via: org.eclipse.swt.tests.gtk.Test_GtkTextEncoding * * About JNI & string conversion: * ############################# * - Regular JNI String conversion usually uses a modified UTF-8, see: https://en.wikipedia.org/wiki/UTF-8#Modified_UTF-8 * - And in JNI, normally (env*)->GetStringUTFChars(..) is used to convert a javaString into a C string. * See: http://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/functions.html#GetStringUTFChars * * However, the modified UTF-8 only works well with C system functions as it doesn't contain embedded nulls * and is null terminated. * * But because the modified UTF-8 only supports up to 3 bytes (and not up to 4 as regular UTF-8), characters * that require 4 bytes (e.g emojos) are not translated properly from Java to C. * * To work around this issue, we convert the Java string to a byte array on the Java side manually and then pass it to C. * See: http://stackoverflow.com/questions/32205446/getting-true-utf-8-characters-in-java-jni * * Note: * Java uses UTF-16 Wide characters internally to represent a string. * C uses UTF-8 Multibyte characters (null terminated) to represent a string. * * About encoding on Linux/Gtk & it's relevance to SWT: * #################################################### * * UTF-* = variable length encoding. * * UTF-8 = minimum is 8 bits, max is 6 bytes, but rarely goes beyond 4 bytes. Gtk & most of web uses this. * UTF-16 = minimum is 16 bits. Java's string are stored this way. * UTF-16 can be * Big Endian : 65 = 00000000 01000001 # Human friendly, reads left to right. * Little Endian : 65 = 01000001 00000000 # Intel x86 and also AMD64 / x86-64 series of processors use the little-endian [1] * # i.e, we in SWT often have to deal with UTF-16 LE * Some terminology: * - "Code point" is the numerical value of unicode character. * - All of UTF-* have the same letter to code-point mapping, * but UTF-8/16/32 have different "back-ends". * * Illustration: * (char) = (code point) = (back end). * A = 65 = 01000001 UTF-8 * = 00000000 01000001 UTF-16 BE * = 01000001 00000000 UTF-16 LE * * - Byte Order Marks (BOM) are a few bytes at the start of a *file* indicating which endianess is used. * Problem: Gtk/webkit often don't give us BOM's. * (further reading *3) * * - We can reliably encode character to a backend (A -> UTF-8/16), but the other way round is * guess work since byte order marks are often missing and UTF-16 bits are technically valid UTF-8. * (see Converter.heuristic for details). * We could improve our heuristic by using something like http://jchardet.sourceforge.net/. * * - Glib has some conversion functions: * g_utf16_to_utf8 * g_utf8_to_utf16 * * - So does java: (e.g null terminated UTF-8) * ("myString" + '\0').getBytes(StandardCharsets.UTF-8) * * - I suggest using Java functions where possible to avoid memory leaks. * (Yes, they happen and are big-pain-in-the-ass to find https://bugs.eclipse.org/bugs/show_bug.cgi?id=533995) * * * Learning about encoding: * ######################### * I suggest the following 3 videos to understand ASCII/UTF-8/UTF-16[LE|BE]/UTF-32 encoding: * Overview: https://www.youtube.com/watch?v=MijmeoH9LT4 * Details: * Part-1: https://www.youtube.com/watch?v=B1Sf1IhA0j4 * Part-2: https://www.youtube.com/watch?v=-oYfv794R9s * Part-3: https://www.youtube.com/watch?v=vLBtrd9Ar28 * * Also read all of this: * http://kunststube.net/encoding/ * and this: * https://www.joelonsoftware.com/2003/10/08/the-absolute-minimum-every-software-developer-absolutely-positively-must-know-about-unicode-and-character-sets-no-excuses/ * * And lastly, good utf-8 reference: https://en.wikipedia.org/wiki/UTF-8#Description * * You should now be a master of encoding. I wish you luck on your journey. * * [1] https://en.wikipedia.org/wiki/Endianness * [2] https://en.wikipedia.org/wiki/Byte_order_mark * [3] BOM's: http://unicode.org/faq/utf_bom.html#BOM */ public final class Converter { public static final byte[] NullByteArray = new byte[1]; public static final byte[] EmptyByteArray = new byte[0]; public static final char[] EmptyCharArray = new char[0]; /** * Convert a "C" multibyte UTF-8 string byte array into a Java UTF-16 Wide character array. * * @param buffer - byte buffer with C bytes representing a string. * @return char array representing the string. Usually used for String construction like: new String(mbcsToWcs(..)) */ public static char[] mbcsToWcs(byte[] buffer) { long[] items_written = new long[1]; long ptr = OS.g_utf8_to_utf16(buffer, buffer.length, null, items_written, null); if (ptr == 0) return EmptyCharArray; int length = (int) items_written[0]; char[] chars = new char[length]; C.memmove(chars, ptr, length * 2); OS.g_free(ptr); return chars; } /** * Convert a Java UTF-16 Wide character string into a C UTF-8 Multibyte byte array. * * This algorithm stops when it finds the first NULL character. I.e, if your Java String has embedded NULL * characters, then the returned string will only go up to the first NULL character. * * * @param string - a regular Java String * @param terminate - if <code>true</code> the byte buffer should be terminated with a null character. * @return byte array that can be passed to a native function. */ public static byte[] wcsToMbcs(String string, boolean terminate) { int length = string.length(); char[] buffer = new char[length]; string.getChars(0, length, buffer, 0); return wcsToMbcs(buffer, terminate); } /** * Given a java String, convert it to a regular null terimnated C string, * to be used when calling a native C function. * @param string A java string. * @return a pointer to a C String. In C, this would be a 'char *' */ public static byte[] javaStringToCString(String string) { return wcsToMbcs(string, true); } /** * This method takes a 'C' pointer (char *) or (gchar *), reads characters up to the terminating symbol '\0' and * converts it into a Java String. * * Note: In SWT we don't use JNI's native String functions because of the 3 vs 4 byte issue explained in Class description. * Instead we pass a character pointer from C to java and convert it to a String in Java manually. * * @param cCharPtr - A char * or a gchar *. Which will be freed up afterwards. * @param freecCharPtr - "true" means free up memory pointed to by cCharPtr. * CAREFUL! If this string is part of a struct (ex GError), and a specialized * free function (like g_error_free(..) is called on the whole struct, then you * should not free up individual struct members with this function, * as otherwise you can get unpredictable behavior). * @return a Java String object. */ public static String cCharPtrToJavaString(long cCharPtr, boolean freecCharPtr) { int length = C.strlen(cCharPtr); byte[] buffer = new byte[length]; C.memmove(buffer, cCharPtr, length); if (freecCharPtr) { OS.g_free(cCharPtr); } return new String(mbcsToWcs(buffer)); } /** * Convert a Java UTF-16 Wide character array into a C UTF-8 Multibyte byte array. * * This algorithm stops when it finds the first NULL character. I.e, if your Java String has embedded NULL * characters, then the returned string will only go up to the first NULL character. * * @param chars - a regular Java String * @param terminate - if <code>true</code> the byte buffer should be terminated with a null character. * @return byte array that can be passed to a native function. */ public static byte[] wcsToMbcs(char[] chars, boolean terminate) { long[] items_read = new long[1], items_written = new long[1]; /* * Note that g_utf16_to_utf8() stops converting * when it finds the first NULL. */ long ptr = OS.g_utf16_to_utf8(chars, chars.length, items_read, items_written, null); if (ptr == 0) return terminate ? NullByteArray : EmptyByteArray; int written = (int) items_written[0]; byte[] bytes = new byte[written + (terminate ? 1 : 0)]; C.memmove(bytes, ptr, written); OS.g_free(ptr); return bytes; } /** * Convert a Java UTF-16 Wide character into a single C UTF-8 Multibyte character * that you can pass to a native function. * @param ch - Java UTF-16 wide character. * @return C UTF-8 Multibyte character. */ public static char wcsToMbcs(char ch) { int key = ch & 0xFFFF; if (key <= 0x7F) return ch; byte[] buffer = wcsToMbcs(new char[] { ch }, false); if (buffer.length == 1) return (char) buffer[0]; if (buffer.length == 2) { return (char) (((buffer[0] & 0xFF) << 8) | (buffer[1] & 0xFF)); } return 0; } /** * Convert C UTF-8 Multibyte character into a Java UTF-16 Wide character. * * @param ch - C Multibyte UTF-8 character * @return Java UTF-16 Wide character */ public static char mbcsToWcs(char ch) { int key = ch & 0xFFFF; if (key <= 0x7F) return ch; byte[] buffer; if (key <= 0xFF) { buffer = new byte[1]; buffer[0] = (byte) key; } else { buffer = new byte[2]; buffer[0] = (byte) ((key >> 8) & 0xFF); buffer[1] = (byte) (key & 0xFF); } char[] result = mbcsToWcs(buffer); if (result.length == 0) return 0; return result[0]; } /** * Given a byte array with unknown encoding, try to decode it via (relatively simple) heuristic. * This is useful when we're not provided the encoding by OS/library.<br> * * Current implementation only supports standard java charsets but can be extended as needed. * This method could be improved by using http://jchardet.sourceforge.net/ <br> * * Run time is O(a * n) where a is a constant that varies depending on the size of input n, but roughly 1-20) * * @param bytes raw bits from the OS. * @return String based on the most pop */ public static String byteToStringViaHeuristic(byte[] bytes) { /* * Technical notes: * - Given a sequence of bytes, UTF-8 and UTF-16 cannot determined deterministically (1*). * - However, UTF-16 has a lot of null bytes when code points are mostly in the 0-255 range (using only 2nd byte), * a byte sequence with many null bytes is likely UTF-16. * - Valid UTF-8 technically can contain null bytes, but it's rare. * * Some times it can get confused if it receives two non-null bytes. e.g (E with two dots on top) = (UTF-16 [01,04]) * It can either mean a valid set of UTF-8 characters or a single UTF-16 character. * This issue typically only occurs for very short sequences 1-5 characters of very special characters). * Improving the heuristic for such corner cases is complicated. We'd have to implement a mechanism * that would be aware of character frequencies and assign a score to the probability of each mapping. * * [1] https://softwareengineering.stackexchange.com/questions/187169/how-to-detect-the-encoding-of-a-file */ // Base cases if ((bytes.length == 0) || (bytes.length == 1 && bytes[0] == 0)) { return ""; } // Test if it's valid UTF-8. // Note, ASCII is a subset of UTF-8. try { CharsetDecoder charDecoder = StandardCharsets.UTF_8.newDecoder(); charDecoder.onMalformedInput(CodingErrorAction.REPORT); charDecoder.onUnmappableCharacter(CodingErrorAction.REPORT); String text = charDecoder.decode(ByteBuffer.wrap(bytes)).toString(); // No exception thrown means that we have valid UTF-8 "bit string". However, valid UTF-8 bit string doesn't mean it's the corect decoding. // We have assert correctness via an educated guess boolean probablyUTF8 = true; { // Problem 1: It might be UTF-16 since at the binary level UTF-16 can be valid UTF-8. (null is a valid utf-8 character). // Solution: Count nulls to try to guess if it's UTF-16. // Verified via // org.eclipse.swt.tests.gtk.Test_GtkConverter.test_HeuristicUTF16_letters() // org.eclipse.swt.tests.gtk.Test_GtkConverter.test_HeuristicUTF16_letter() double nullBytePercentageForUtf16 = 0.01; // if more than this % null bytes, then it's probably utf-16. int nullCount = 0; for (byte b : bytes) { if (b == 0) nullCount++; } double nullPercentage = (double) nullCount / (double) bytes.length; if (nullPercentage > nullBytePercentageForUtf16) { probablyUTF8 = false; } } // Problem 2: Valid UTF-8 bit string can map to invalid code points (i.e undefined unicode) // Solution 2: verify that every character is a valid code point. if (probablyUTF8) { char[] chars = text.toCharArray(); for (int i = 0; i < chars.length; i++) { int codePoint = Character.codePointAt(chars, i); if (!Character.isValidCodePoint(codePoint)) { probablyUTF8 = false; break; } } } // Problem 3: Short 2-byte sequences are very ambiguous. // E.g Unicode Hyphen U+2010 (which looks like a '-') ( which btw different from the ascii U+002D '-' Hyphen-Minus) // can be miss-understood as 16 (Synchronous Idle) & 32 (Space). // Solution: Unless we have two valid alphabet characters, it's probably a single utf-16 character. // However, this leads to the problem that single non-alphabetic unicode characters are not recognized correctly. // Below code is left in case recognizing alphabetic characters is of higher priority than exotic unicode once. // if (probablyUTF8) { // if (bytes.length == 2) { // char [] chars = text.toCharArray(); // for (int i = 0; i < chars.length; i++) { // int codePoint = Character.codePointAt(chars, i); // if (!Character.isAlphabetic(codePoint)) { // probablyUTF8 = false; // break; // } // } // } // } if (!probablyUTF8) { return new String(bytes, StandardCharsets.UTF_16LE); } else { return text; } } catch (CharacterCodingException e) { } // Invalid UTF-8. Try other character sets. Charset[] commonWebCharSets = new Charset[] { StandardCharsets.UTF_16LE, StandardCharsets.ISO_8859_1, StandardCharsets.UTF_16BE, StandardCharsets.UTF_16 }; for (Charset setToTry : commonWebCharSets) { try { CharsetDecoder charDecoder = setToTry.newDecoder(); charDecoder.onMalformedInput(CodingErrorAction.REPORT); charDecoder.onUnmappableCharacter(CodingErrorAction.REPORT); return charDecoder.decode(ByteBuffer.wrap(bytes)).toString(); } catch (CharacterCodingException e) { } } // Could not determine encoding. // Return error string with stack trace to help users determine which function lead to a failed decoding. StringWriter sw = new StringWriter(); new Throwable("").printStackTrace(new PrintWriter(sw)); return "SWT: Failed to decode byte buffer. Encoding is not ASCII/UTF-8/UTF-16[LE|BE|BOM]/ISO_8859_1. Stack trace:\n" + sw.toString(); } }