Example usage for java.nio.charset StandardCharsets UTF

Introduction

In this page you can find the example usage for java.nio.charset StandardCharsets UTF_8.

Prototype

Charset UTF_8

To view the source code for java.nio.charset StandardCharsets UTF_8.

Click Source Link

Document

Eight-bit UCS Transformation Format.

Usage

From source file:com.microsoft.azure.management.datalake.store.uploader.StringExtensions.java

/**
 * Finds the index in the given buffer of a newline character, either the first or the last (based on the parameters).
 * If a combined newline (\r\n), the index returned is that of the last character in the sequence.
 *
 * @param buffer The buffer to search in.
 * @param startOffset The index of the first byte to start searching at.
 * @param length The number of bytes to search, starting from the given startOffset.
 * @param reverse If true, searches from the startOffset down to the beginning of the buffer. If false, searches upwards.
 * @param encoding Indicates the type of encoding to use for the buffered bytes.
 * @param delimiter Optionally indicates the delimiter to consider as the "new line", which MUST BE a single character. If null, the default is '\\r', '\\n' and '\\r\\n'.
 * @return The index of the closest newline character in the sequence (based on direction) that was found. Returns -1 if not found.
 *//*  w w w.ja va  2s  .c  o  m*/
public static int findNewline(byte[] buffer, int startOffset, int length, boolean reverse, Charset encoding,
        String delimiter) {
    if (buffer.length == 0 || length == 0) {
        return -1;
    }

    // define the bytes per character to use
    int bytesPerChar;
    if (encoding.equals(StandardCharsets.UTF_16) || encoding.equals(StandardCharsets.UTF_16BE)
            || encoding.equals(StandardCharsets.UTF_16LE)) {
        bytesPerChar = 2;
    } else if (encoding.equals(StandardCharsets.US_ASCII) || encoding.equals(StandardCharsets.UTF_8)) {
        bytesPerChar = 1;
    } else {
        throw new IllegalArgumentException(
                "Only the following encodings are allowed: UTF-8, UTF-16, UTF-16BE, UTF16-LE and ASCII");
    }

    if (delimiter != null && !StringUtils.isEmpty(delimiter) && delimiter.length() > 1) {
        throw new IllegalArgumentException(
                "The delimiter must only be a single character or unspecified to represent the CRLF delimiter");
    }

    if (delimiter != null && !StringUtils.isEmpty(delimiter)) {
        // convert the byte array back to a String
        int startOfSegment = reverse ? startOffset - length + 1 : startOffset;
        String bytesToString = new String(buffer, startOfSegment, length, encoding);
        if (!bytesToString.contains(delimiter)) {
            // didn't find the delimiter.
            return -1;
        }

        // the index is returned, which is 0 based, so our loop must include the zero case.
        int numCharsToDelim = reverse ? bytesToString.lastIndexOf(delimiter) : bytesToString.indexOf(delimiter);
        int toReturn = 0;
        for (int i = 0; i <= numCharsToDelim; i++) {
            toReturn += Character.toString(bytesToString.charAt(startOfSegment + i)).getBytes(encoding).length;
        }

        // we get the total number of bytes, but we want to return the index (which starts at 0)
        // so we subtract 1 from the total number of bytes to get the final byte index.
        return toReturn - 1;
    }

    //endOffset is a 'sentinel' value; we use that to figure out when to stop searching
    int endOffset = reverse ? startOffset - length : startOffset + length;

    // if we are starting at the end, we need to move toward the front enough to grab the right number of bytes
    startOffset = reverse ? startOffset - (bytesPerChar - 1) : startOffset;

    if (startOffset < 0 || startOffset >= buffer.length) {
        throw new IndexOutOfBoundsException(
                "Given start offset is outside the bounds of the given buffer. In reverse cases, the start offset is modified to ensure we check the full size of the last character");
    }

    // make sure that the length we are traversing is at least as long as a single character
    if (length < bytesPerChar) {
        throw new IllegalArgumentException(
                "length must be at least as long as the length, in bytes, of a single character");
    }

    if (endOffset < -1 || endOffset > buffer.length) {
        throw new IndexOutOfBoundsException(
                "Given combination of startOffset and length would execute the search outside the bounds of the given buffer.");
    }

    int bufferEndOffset = reverse ? startOffset : startOffset + length;
    int result = -1;
    for (int charPos = startOffset; reverse ? charPos != endOffset
            : charPos + bytesPerChar - 1 < endOffset; charPos = reverse ? charPos - 1 : charPos + 1) {
        char c;
        if (bytesPerChar == 1) {
            c = (char) buffer[charPos];
        } else {
            String temp = new String(buffer, charPos, bytesPerChar, encoding);
            if (StringUtils.isEmpty(temp)) {
                continue;
            } else {
                c = temp.toCharArray()[0];
            }
        }

        if (isNewline(c, delimiter)) {
            result = charPos + bytesPerChar - 1;
            break;
        }
    }

    if ((delimiter == null || StringUtils.isEmpty(delimiter)) && !reverse
            && result < bufferEndOffset - bytesPerChar) {
        char c;
        if (bytesPerChar == 1) {
            c = (char) buffer[result + bytesPerChar];
        } else {
            String temp = new String(buffer, result + 1, bytesPerChar, encoding);
            if (StringUtils.isEmpty(temp)) {
                // this can occur if the number of bytes for characters in the string result in an empty string (an invalid code for the given encoding)
                // in this case, that means that we are done for the default delimiter.
                return result;
            } else {
                c = temp.toCharArray()[0];
            }
        }

        if (isNewline(c, delimiter)) {
            //we originally landed on a \r character; if we have a \r\n character, advance one position to include that
            result += bytesPerChar;
        }
    }

    return result;
}

From source file:com.streamsets.pipeline.lib.io.TestLiveFileChunk.java

@Test
public void testChunkGetters() throws IOException {
    LiveFile file = Mockito.mock(LiveFile.class);
    LiveFileChunk chunk = new LiveFileChunk("tag", file, StandardCharsets.UTF_8, "Hola\nHello".getBytes(), 1, 9,
            true);//from  ww  w. j  a v  a2s.  c  om
    Assert.assertEquals("tag", chunk.getTag());
    Assert.assertEquals(file, chunk.getFile());
    Assert.assertEquals("Hola", IOUtils.readLines(chunk.getReader()).get(0));
    Assert.assertEquals("Hell", IOUtils.readLines(chunk.getReader()).get(1));
    Assert.assertEquals(1, chunk.getOffset());
    Assert.assertEquals(9, chunk.getLength());
    Assert.assertTrue(chunk.isTruncated());
    Assert.assertEquals(2, chunk.getLines().size());
    Assert.assertEquals("Hola\n", chunk.getLines().get(0).getText());
    Assert.assertEquals(1, chunk.getLines().get(0).getFileOffset());
    Assert.assertEquals(chunk.getBuffer(), chunk.getLines().get(0).getBuffer());
    Assert.assertEquals(0, chunk.getLines().get(0).getOffset());
    Assert.assertEquals(5, chunk.getLines().get(0).getLength());
    Assert.assertEquals("Hell", chunk.getLines().get(1).getText());
    Assert.assertEquals(6, chunk.getLines().get(1).getFileOffset());
    Assert.assertEquals(chunk.getBuffer(), chunk.getLines().get(1).getBuffer());
    Assert.assertEquals(5, chunk.getLines().get(1).getOffset());
    Assert.assertEquals(4, chunk.getLines().get(1).getLength());
}

From source file:com.haulmont.cuba.gui.xml.layout.LayoutLoader.java

public static Document parseDescriptor(InputStream stream) {
    checkNotNullArgument(stream, "Input stream is null");

    String template;/*from w w w .j  a v a 2  s  .c  o m*/
    try {
        template = IOUtils.toString(stream, StandardCharsets.UTF_8);
    } catch (IOException e) {
        throw new IllegalStateException(e);
    }

    return parseDescriptor(template);
}

From source file:com.thinkbiganalytics.discovery.util.ParserHelperTest.java

public void textExtract(String text, int numRows, int numExpected) throws Exception {
    try (InputStream is = new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8))) {
        String value = ParserHelper.extractSampleLines(is, StandardCharsets.UTF_8, numRows);
        assertEquals(numExpected, value.split("\n").length);
    }/*from w  w  w  .  j a  v  a 2 s .co m*/
}

From source file:com.amazon.kinesis.streaming.agent.processing.processors.SingleLineDataConverter.java

@Override
public ByteBuffer convert(ByteBuffer data) throws DataConversionException {
    String dataStr = ByteBuffers.toString(data, StandardCharsets.UTF_8);
    String[] lines = dataStr.split(NEW_LINE);
    for (int i = 0; i < lines.length; i++) {
        // FIXME: Shall we trim each line?
        lines[i] = lines[i].trim();/*from   w w  w  . j av a2 s. co m*/
    }

    String dataRes = StringUtils.join(lines) + NEW_LINE;
    return ByteBuffer.wrap(dataRes.getBytes(StandardCharsets.UTF_8));
}

From source file:com.thoughtworks.go.config.IdFileService.java

public String load() {
    try {//from   ww w.  ja  v a  2s .co  m
        return FileUtils.readFileToString(file, StandardCharsets.UTF_8).trim();
    } catch (IOException ioe) {
        throw bomb(String.format("Couldn't load %s from filesystem", file.getName()), ioe);
    }
}

From source file:com.haulmont.cuba.core.sys.javacl.SourceProvider.java

public String getSourceString(String name) throws IOException {
    File srcFile = getSourceFile(name);
    if (!srcFile.exists()) {
        throw new FileNotFoundException(String.format("Java source for %s not found", name));
    }/*from  www . j  av a2 s  . co  m*/
    return FileUtils.readFileToString(srcFile, StandardCharsets.UTF_8);
}

From source file:com.hurence.logisland.processor.hbase.util.StringSerDe.java

@Override
public String deserialize(final InputStream input) throws DeserializationException, IOException {
    byte[] value = IOUtils.toByteArray(input);
    if (value == null) {
        return null;
    }//w w  w  . j  a va  2s.  c om

    return new String(value, StandardCharsets.UTF_8);
}

From source file:com.spectralogic.ds3client.utils.MetadataStringManipulation.java

/**
 * Percent encodes non-alpha-numeric characters within the specified string
 * excluding the symbols listed in {@link #HTTP_HEADER_SAFE_CHARS} using UTF-8
 */// w ww  .j  av a 2 s  .  c  om
public static String toEncodedString(final String str) {
    if (str == null) {
        return null;
    }

    final String strUtf8 = new String(str.getBytes(StandardCharsets.UTF_8), StandardCharsets.UTF_8);
    return getMetadataEscaper().escape(strUtf8);
}

From source file:eu.itesla_project.computation.mpi.ExportTasksStatisticsTool.java

@Override
public void run(CommandLine line) throws Exception {
    Path statisticsDbDir = Paths.get(line.getOptionValue("statistics-db-dir"));
    String statisticsDbName = line.getOptionValue("statistics-db-name");
    Path outputFile = Paths.get(line.getOptionValue("output-file"));
    try (MpiStatistics statistics = new CsvMpiStatistics(statisticsDbDir, statisticsDbName)) {
        try (Writer writer = Files.newBufferedWriter(outputFile, StandardCharsets.UTF_8)) {
            statistics.exportTasksToCsv(writer);
        }//from ww w  .j  a v  a2 s  .  c  o m
    }
}

Example usage for java.nio.charset StandardCharsets UTF_8

Introduction

Prototype

Document

Usage