Example usage for org.apache.hadoop.io Text set

Introduction

In this page you can find the example usage for org.apache.hadoop.io Text set.

Prototype

public void set(Text other) 

Source Link

Document

copy a text.

Usage

From source file:com.datatorrent.contrib.hdht.HadoopFilePerformanceTest.java

License:Open Source License

@Test
public void testMapFileRead() throws Exception {

    logger.info("Reading {} with {} key/value pairs", Testfile.MAPFILE.filename(),
            String.format("%,d", testSize));
    writeMapFile();/*ww w  . java 2 s .c  o  m*/

    Text key = new Text();
    Text value = new Text();

    writeMapFile();

    // Set amount of memory to use for buffer
    float bufferPercent = 0.25f;
    int bufferSize = (int) (ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax() * bufferPercent);

    MapFile.Reader reader = new MapFile.Reader(Testfile.MAPFILE.filepath(), conf,
            SequenceFile.Reader.bufferSize(bufferSize));

    startTimer();
    reader.reset();
    while (reader.next(key, value)) {
        //logger.debug("read key:{} value:{}", key, value);
    }
    logger.info("Duration for reader.next() SEQUENTIAL keys: {}", stopTimer(Testfile.MAPFILE, "READ-SEQ"));

    startTimer();
    reader.reset();
    for (int i = 0; i < testSize; i++) {
        key.set(getKey(i));
        reader.get(key, value);
        //logger.debug("{}:{}", key, value);
    }
    logger.info("Duration for reader.get(key) SEQUENTIAL keys: {}", stopTimer(Testfile.MAPFILE, "READ-SEQ-ID"));

    Random random = new Random();
    startTimer();
    for (int i = 0; i < testSize; i++) {
        key.set(getKey(random.nextInt(testSize)));
        reader.get(key, value);
        //logger.debug("{}:{}", key, value);
    }
    logger.info("Duration for reader.get(key) RANDOM keys: {}", stopTimer(Testfile.MAPFILE, "READ-RAND"));
    reader.close();

}

---

From source file:com.davidgildeh.hadoop.input.simpledb.SimpleDBRecordReader.java

License:Apache License

/**
 * Get next Key/Value Record (Tuple) from the Split
 * /* ww w .  j  a  v  a2  s.c om*/
 * @param key           The key to set
 * @param value         The HashMap value to set
 * @return              True - next Item available, False - No more items available
 * @throws IOException 
 */
public boolean next(Text key, MapWritable value) throws IOException {

    // Get next item off the ArrayList unless we're at the end
    if (cursor < split.getLength()) {

        Item item = items.get(cursor++);

        key.set(item.getName());
        for (Attribute attribute : item.getAttributes()) {
            value.put(new Text(attribute.getName()), new Text(attribute.getValue()));
        }

        if (LOG.isDebugEnabled()) {
            LOG.debug("Sending next record to Mappers: " + key.toString());
        }

        return true;
    } else {
        return false;
    }
}

---

From source file:com.ebay.erl.mobius.core.model.TupleTest.java

License:Apache License

private Tuple prepareTupe() {
    Tuple t = new Tuple();

    t.put("C1", 1);
    t.put("C2", 1.2F);
    t.put("C3", 1.3D);
    t.put("C4", 1L);
    t.put("C5", "s5");

    CaseInsensitiveTreeMap m = new CaseInsensitiveTreeMap();
    m.put("k1", "v1");
    m.put("k2", "v2");

    t.put("C6", m);

    Text txt = new Text();
    txt.set("hello");
    t.put("C7", txt);

    BigDecimal serializable = new BigDecimal(1000);
    t.put("C8", serializable);

    t.put("C9", getNull());

    t.put("C10", false);

    t.put("C11", java.sql.Date.valueOf("2000-12-12"));

    t.put("C12", (byte) 0x09);

    return t;/*from  w w  w .  j  a v a 2s  . c  om*/
}

---

From source file:com.ebay.nest.io.sede.objectinspector.primitive.WritableStringObjectInspector.java

License:Apache License

@Override
public Object create(Text value) {
    Text r = new Text();
    if (value != null) {
        r.set(value);
    }/*from   w ww  . j  a  va  2  s.  c  o  m*/
    return r;
}

---

From source file:com.ebay.nest.io.sede.objectinspector.primitive.WritableStringObjectInspector.java

License:Apache License

@Override
public Object create(String value) {
    Text r = new Text();
    if (value != null) {
        r.set(value);
    }/*from  w ww  .j av  a  2 s  .c  om*/
    return r;
}

---

From source file:com.ebay.nest.io.sede.objectinspector.primitive.WritableStringObjectInspector.java

License:Apache License

@Override
public Object set(Object o, Text value) {
    Text r = (Text) o;
    if (value != null) {
        r.set(value);
    }//from w  w  w . j av  a  2  s.  c  om
    return o;
}

---

From source file:com.ebay.nest.io.sede.objectinspector.primitive.WritableStringObjectInspector.java

License:Apache License

@Override
public Object set(Object o, String value) {
    Text r = (Text) o;
    if (value != null) {
        r.set(value);
    }/*from   w w w  .  j  a v  a 2 s  .  c om*/
    return o;
}

---

From source file:com.endgame.binarypig.util.BuildSequenceFileFromArchive.java

License:Apache License

public void load(FileSystem fs, Configuration conf, File archive, Path outputDir) throws Exception {
    Text key = new Text();
    BytesWritable val = new BytesWritable();

    SequenceFile.Writer writer = null;
    ArchiveInputStream archiveInputStream = null;

    try {//from w  w w.j  a  v  a2s  .  c  o m
        Path sequenceName = new Path(outputDir, archive.getName() + ".seq");
        System.out.println("Writing to " + sequenceName);
        writer = SequenceFile.createWriter(fs, conf, sequenceName, Text.class, BytesWritable.class,
                CompressionType.RECORD);
        String lowerName = archive.toString().toLowerCase();

        if (lowerName.endsWith(".tar.gz") || lowerName.endsWith(".tgz")) {
            archiveInputStream = new ArchiveStreamFactory().createArchiveInputStream("tar",
                    new GZIPInputStream(new FileInputStream(archive)));
        } else if (lowerName.endsWith(".tar.bz") || lowerName.endsWith(".tar.bz2")
                || lowerName.endsWith(".tbz")) {
            FileInputStream is = new FileInputStream(archive);
            is.read(); // read 'B'
            is.read(); // read 'Z'
            archiveInputStream = new ArchiveStreamFactory().createArchiveInputStream("tar",
                    new CBZip2InputStream(is));
        } else if (lowerName.endsWith(".tar")) {
            archiveInputStream = new ArchiveStreamFactory().createArchiveInputStream("tar",
                    new FileInputStream(archive));
        } else if (lowerName.endsWith(".zip")) {
            archiveInputStream = new ArchiveStreamFactory().createArchiveInputStream("zip",
                    new FileInputStream(archive));
        } else {
            throw new RuntimeException("Can't handle archive format for: " + archive);
        }

        ArchiveEntry entry = null;
        while ((entry = archiveInputStream.getNextEntry()) != null) {
            if (!entry.isDirectory()) {
                try {
                    byte[] outputFile = IOUtils.toByteArray(archiveInputStream);
                    val.set(outputFile, 0, outputFile.length);
                    key.set(DigestUtils.md5Hex(outputFile));

                    writer.append(key, val);
                } catch (IOException e) {
                    System.err.println("Warning: archive may be truncated: " + archive);
                    // Truncated Archive
                    break;
                }
            }
        }
    } finally {
        archiveInputStream.close();
        writer.close();
    }
}

---

From source file:com.endgame.binarypig.util.BuildSequenceFileFromDir.java

License:Apache License

@Override
public int run(String[] args) throws Exception {

    File inDir = new File(args[0]);
    Path name = new Path(args[1]);

    Text key = new Text();
    BytesWritable val = new BytesWritable();

    Configuration conf = getConf();
    FileSystem fs = FileSystem.get(conf);
    SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, name, Text.class, BytesWritable.class,
            CompressionType.RECORD);/*from ww w . ja v  a2s  . c  o  m*/

    for (File file : inDir.listFiles()) {
        if (!file.isFile()) {
            System.out.println("Skipping " + file + " (not a file) ...");
            continue;
        }

        byte[] bytes = FileUtils.readFileToByteArray(file);
        val.set(bytes, 0, bytes.length);
        key.set(DigestUtils.md5Hex(bytes));
        writer.append(key, val);
    }
    writer.close();

    return 0;
}

---

From source file:com.facebook.hive.orc.lazy.LazyStringDirectTreeReader.java

License:Open Source License

@Override
public Object next(Object previous) throws IOException {
    Text result = null;
    if (valuePresent) {
        if (previous == null) {
            result = new Text();
        } else {//from  w ww .  j  ava2 s .c  om
            result = (Text) previous;
        }
        int len = (int) lengths.next();
        int offset = 0;
        byte[] bytes = new byte[len];
        while (len > 0) {
            int written = stream.read(bytes, offset, len);
            if (written < 0) {
                throw new EOFException("Can't finish byte read from " + stream);
            }
            len -= written;
            offset += written;
        }
        result.set(bytes);
    }
    return result;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• »