Example usage for org.apache.hadoop.io Text getBytes

Introduction

In this page you can find the example usage for org.apache.hadoop.io Text getBytes.

Prototype

@Override
public byte[] getBytes()

Source Link

Document

Returns the raw bytes; however, only data up to #getLength() is valid.

Usage

From source file:io.aos.hdfs.TextTest.java

License:Apache License

@Test
public void mutability() throws IOException {
    // vv TextTest-Mutability
    Text t = new Text("hadoop");
    t.set("pig");
    assertThat(t.getLength(), is(3));//  w ww.jav  a 2s. c  om
    assertThat(t.getBytes().length, is(3));
    // ^^ TextTest-Mutability
}

From source file:io.aos.hdfs.TextTest.java

License:Apache License

@Test
public void byteArrayNotShortened() throws IOException {
    // vv TextTest-ByteArrayNotShortened
    Text t = new Text("hadoop");
    t.set(/*[*/new Text("pig")/*]*/);
    assertThat(t.getLength(), is(3));/* www  .  j  av a2s  .co m*/
    assertThat("Byte length not shortened", t.getBytes().length, /*[*/is(6)/*]*/);
    // ^^ TextTest-ByteArrayNotShortened
}

From source file:io.covert.dns.extract.ExtractorMapper.java

License:Apache License

protected void map(Text jsonRecord, Text empty, Context context)
        throws java.io.IOException, InterruptedException {
    Map<String, Object> record = objectMapper.readValue(jsonRecord.getBytes(),
            new TypeReference<Map<String, Object>>() {
            });/*from  w w w.  jav  a  2s  .  c o m*/
    Object result = extractor.extract(record);
    if (result == null) {
        context.getCounter(ExtractorMapper.class.getSimpleName(), "NO RESULT").increment(1);
    } else {
        context.getCounter(ExtractorMapper.class.getSimpleName(), "RESULT").increment(1);
        String val = objectMapper.writeValueAsString(result);
        if (stripOuterQuotes)
            val = val.replaceAll("^\"+", "").replaceAll("\"+$", "");
        outKey.set(val);
        context.write(outKey, empty);
    }
    context.progress();
}

From source file:io.covert.dns.filtering.FilterMapper.java

License:Apache License

protected void map(Text jsonRecord, Text empty, Context context)
        throws java.io.IOException, InterruptedException {
    Map<String, Object> record = objectMapper.readValue(jsonRecord.getBytes(),
            new TypeReference<Map<String, Object>>() {
            });/*from   w w w . j a  v a2s  .co  m*/
    if (filter.filter(record)) {
        context.getCounter(FilterMapper.class.getSimpleName(), MATCHED).increment(1);
        context.write(jsonRecord, empty);
    } else {
        context.getCounter(FilterMapper.class.getSimpleName(), NON_MATCHED).increment(1);
    }
    context.progress();
}

From source file:io.covert.dns.geo.GeoMapper.java

License:Apache License

protected void map(Text json, Text empty, Context context) throws java.io.IOException, InterruptedException {
    Map<String, Object> rec = objectMapper.readValue(json.getBytes(), new TypeReference<Map<String, Object>>() {
    });/* w w w  .  j a  v  a2s .  co m*/
    String ip = (String) rec.get("addr");
    if (ip != null) {
        // annotate with IP Geo info...
        Map<String, Object> loc = geo.getLocation(ip);
        rec.putAll(loc);
        outKey.set(objectMapper.writeValueAsBytes(rec));
        context.write(outKey, empty);
    } else {
        // No Geo, but pass through anyway
        context.write(json, empty);
    }
}

From source file:io.covert.dns.storage.StorageMapper.java

License:Apache License

protected void map(Text jsonRecord, Text empty, Context context)
        throws java.io.IOException, InterruptedException {
    Map<String, Object> record;
    try {// w  ww  .j  a v  a 2 s  .c o  m
        record = objectMapper.readValue(jsonRecord.getBytes(), new TypeReference<Map<String, Object>>() {
        });
    } catch (Exception e) {
        context.getCounter("ERROR:" + e.getClass().getSimpleName(), "BAD_RECORDS").increment(1);
        return;
    }

    try {
        storageModule.store(record);
        context.getCounter(storageModule.getClass().getSimpleName(), "RECORDS_STORED").increment(1);
    } catch (Exception e) {
        context.getCounter("ERROR:" + e.getClass().getSimpleName(),
                "FAILED_STORE:" + storageModule.getClass().getSimpleName()).increment(1);
    }
    context.getCounter("TOTAL", "RECORDS").increment(1);
    context.progress();
}

From source file:io.fluo.core.util.ByteUtil.java

License:Apache License

/**
 * Convert from Hadoop Text to Bytes object
 * // w  w w . j  ava2 s  .com
 * @param t Text
 * @return Bytes object
 */
public static Bytes toBytes(Text t) {
    return Bytes.wrap(t.getBytes(), 0, t.getLength());
}

From source file:it.crs4.pydoop.mapreduce.pipes.BinaryProtocol.java

License:Apache License

/**
 * Write the given object to the stream. If it is a Text or BytesWritable,
 * write it directly. Otherwise, write it to a buffer and then write the
 * length and data to the stream./*from  ww  w.  j a  v a 2s. c om*/
 * @param obj the object to write
 * @throws IOException
 */
private void writeObject(Writable obj) throws IOException {
    // For Text and BytesWritable, encode them directly, so that they end up
    // in C++ as the natural translations.
    if (obj instanceof Text) {
        Text t = (Text) obj;
        int len = t.getLength();
        WritableUtils.writeVInt(stream, len);
        stream.write(t.getBytes(), 0, len);
    } else if (obj instanceof BytesWritable) {
        BytesWritable b = (BytesWritable) obj;
        int len = b.getLength();
        WritableUtils.writeVInt(stream, len);
        stream.write(b.getBytes(), 0, len);
    } else if (obj == null) {
        // write a zero length string
        WritableUtils.writeVInt(stream, 0);
    } else {
        buffer.reset();
        obj.write(buffer);
        int length = buffer.getLength();
        WritableUtils.writeVInt(stream, length);
        stream.write(buffer.getData(), 0, length);
    }
}

From source file:it.crs4.pydoop.mapreduce.pipes.CommonStub.java

License:Apache License

protected void writeObject(Writable obj, DataOutputStream stream) throws IOException {
    // For Text and BytesWritable, encode them directly, so that they end up
    // in C++ as the natural translations.
    System.err.println("obj: " + obj);

    DataOutputBuffer buffer = new DataOutputBuffer();
    if (obj instanceof Text) {
        Text t = (Text) obj;
        int len = t.getLength();
        WritableUtils.writeVLong(stream, len);
        stream.flush();/*from ww w  .java2  s  .  c o  m*/

        stream.write(t.getBytes(), 0, len);
        stream.flush();
        System.err.println("len: " + len);

    } else if (obj instanceof BytesWritable) {
        BytesWritable b = (BytesWritable) obj;
        int len = b.getLength();
        WritableUtils.writeVLong(stream, len);
        stream.write(b.getBytes(), 0, len);
        System.err.println("len: " + len);
    } else {
        buffer.reset();
        obj.write(buffer);
        int length = buffer.getLength();
        WritableUtils.writeVInt(stream, length);
        stream.write(buffer.getData(), 0, length);
        System.err.println("len: " + length);
    }
    stream.flush();

}

From source file:it.crs4.pydoop.pipes.BinaryProtocol.java

License:Apache License

/**
 * Write the given object to the stream. If it is a Text or BytesWritable,
 * write it directly. Otherwise, write it to a buffer and then write the
 * length and data to the stream./* w ww .  j av a2s.c  o m*/
 * @param obj the object to write
 * @throws IOException
 */
private void writeObject(Writable obj) throws IOException {
    // For Text and BytesWritable, encode them directly, so that they end up
    // in C++ as the natural translations.
    if (obj instanceof Text) {
        Text t = (Text) obj;
        int len = t.getLength();
        WritableUtils.writeVInt(stream, len);
        stream.write(t.getBytes(), 0, len);
    } else if (obj instanceof BytesWritable) {
        BytesWritable b = (BytesWritable) obj;
        int len = b.getLength();
        WritableUtils.writeVInt(stream, len);
        stream.write(b.getBytes(), 0, len);
    } else {
        buffer.reset();
        obj.write(buffer);
        int length = buffer.getLength();
        WritableUtils.writeVInt(stream, length);
        stream.write(buffer.getData(), 0, length);
    }
}