Example usage for org.apache.hadoop.io Text set

Introduction

In this page you can find the example usage for org.apache.hadoop.io Text set.

Prototype

public void set(Text other)

Source Link

Document

copy a text.

Usage

From source file:com.asakusafw.example.direct.seqfile.jobflow.format.CategorySummarySeqFileFormat.java

License:Apache License

@Override
protected void copyFromModel(CategorySummary model, Text key, SummaryWritable value) throws IOException {
    key.set(model.getCategoryCode());
    value.setAmountTotal(model.getAmountTotal());
    value.setSellingPriceTotal(model.getSellingPriceTotal());
}

From source file:com.asakusafw.example.direct.seqfile.jobflow.format.ItemInfoSeqFileFormat.java

License:Apache License

@Override
protected void copyFromModel(ItemInfo model, Text key, ItemInfoWritable value) throws IOException {
    key.set(model.getItemCode());
    value.setItemName(model.getItemNameAsString());
    value.setDepartmentCode(model.getDepartmentCodeOption().or(""));
    value.setDepartmentName(model.getDepartmentNameOption().or(""));
    value.setCategoryCode(model.getCategoryCodeOption().or(""));
    value.setCategoryName(model.getCategoryNameOption().or(""));
    value.setUnitSellingPrice(model.getUnitSellingPriceOption().or(0));
    value.setRegisteredDate(toUtilDate(model.getRegisteredDateOption()));
    value.setBeginDate(toUtilDate(model.getBeginDateOption()));
    value.setEndDate(toUtilDate(model.getEndDateOption()));
}

From source file:com.asakusafw.example.direct.seqfile.jobflow.format.StoreInfoSeqFileFormat.java

License:Apache License

@Override
protected void copyFromModel(StoreInfo model, Text key, Text value) throws IOException {
    key.set(model.getStoreCode());
    value.set(model.getStoreName());/*from  www .  j  a v a 2  s  .  c o m*/
}

From source file:com.asakusafw.lang.compiler.extension.testdriver.mock.MockTextDefinition.java

License:Apache License

@Override
public Text toObject(DataModelReflection reflection) {
    Text text = new Text();
    String string = (String) reflection.getValue(VALUE);
    if (string != null) {
        text.set(string);
    }// w  w  w  . ja  v a  2 s  .c  o m
    return text;
}

From source file:com.asakusafw.runtime.directio.hadoop.SequenceFileFormatTest.java

License:Apache License

/**
 * Test for input.//ww  w.j a v  a 2  s  .c om
 * @throws Exception if failed
 */
@Test
public void input() throws Exception {
    final int count = 10000;
    LocalFileSystem fs = FileSystem.getLocal(conf);
    Path path = new Path(folder.newFile("testing").toURI());
    try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path, LongWritable.class,
            Text.class)) {
        LongWritable k = new LongWritable();
        Text v = new Text();
        for (int i = 0; i < count; i++) {
            k.set(i);
            v.set("Hello, world at " + i);
            writer.append(k, v);
        }
    }

    try (ModelInput<StringOption> in = format.createInput(StringOption.class, fs, path, 0,
            fs.getFileStatus(path).getLen(), new Counter())) {
        StringOption value = new StringOption();
        for (int i = 0; i < count; i++) {
            String answer = "Hello, world at " + i;
            assertThat(answer, in.readTo(value), is(true));
            assertThat(value.getAsString(), is(answer));
        }
        assertThat("eof", in.readTo(value), is(false));
    }
}

From source file:com.asakusafw.runtime.directio.hadoop.SequenceFileFormatTest.java

License:Apache License

/**
 * Test for input.//w w w . ja  v  a  2 s  . co m
 * @throws Exception if failed
 */
@Test
public void input_fragment() throws Exception {
    final int count = 30000;
    Random rand = new Random();
    LocalFileSystem fs = FileSystem.getLocal(conf);
    Path path = new Path(folder.newFile("testing").toURI());
    try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path, LongWritable.class,
            Text.class)) {
        LongWritable k = new LongWritable();
        Text v = new Text();
        for (int i = 0; i < count; i++) {
            k.set(i);
            v.set("Hello, world at " + i);
            writer.append(k, v);
        }
    }

    long fileLen = fs.getFileStatus(path).getLen();
    StringOption value = new StringOption();
    for (int attempt = 0; attempt < 5; attempt++) {
        int index = 0;
        long offset = 0;
        while (offset < fileLen) {
            long length = SequenceFile.SYNC_INTERVAL * (rand.nextInt(10) + 2);
            length = Math.min(length, fileLen - offset);
            try (ModelInput<StringOption> in = format.createInput(StringOption.class, fs, path, offset, length,
                    new Counter())) {
                while (in.readTo(value)) {
                    String answer = "Hello, world at " + index;
                    assertThat(value.getAsString(), is(answer));
                    index++;
                }
                assertThat("eof", in.readTo(value), is(false));
            }
            offset += length;
        }
        assertThat(index, is(count));
    }
}

From source file:com.asakusafw.runtime.directio.hadoop.SequenceFileFormatTest.java

License:Apache License

/**
 * Test for input.//from ww w.j  a va 2s.co  m
 * @throws Exception if failed
 */
@Test
public void input_largerecord() throws Exception {
    StringBuilder buf = new StringBuilder();
    for (int i = 0; i < 1000000; i++) {
        buf.append("Hello, world!");
    }
    Text record = new Text(buf.toString());

    final int count = 5;
    LocalFileSystem fs = FileSystem.getLocal(conf);
    Path path = new Path(folder.newFile("testing").toURI());
    try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path, LongWritable.class,
            Text.class)) {
        LongWritable k = new LongWritable();
        Text v = new Text();
        for (int i = 0; i < count; i++) {
            k.set(i);
            v.set(record);
            writer.append(k, v);
        }
    }

    long fileLen = fs.getFileStatus(path).getLen();
    StringOption value = new StringOption();
    int index = 0;
    long offset = 0;
    while (offset < fileLen) {
        long length = SequenceFile.SYNC_INTERVAL * 2;
        length = Math.min(length, fileLen - offset);
        try (ModelInput<StringOption> in = format.createInput(StringOption.class, fs, path, offset, length,
                new Counter())) {
            while (in.readTo(value)) {
                assertThat(value.get(), is(record));
                index++;
            }
            assertThat("eof", in.readTo(value), is(false));
        }
        offset += length;
    }
    assertThat(index, is(count));
}

From source file:com.asakusafw.runtime.io.sequencefile.SequenceFileUtilTest.java

License:Apache License

/**
 * Reads a sequence file.// w w  w.ja  v a2s  .c om
 * @throws Exception if failed
 */
@Test
public void read() throws Exception {
    Path path = new Path("testing");

    Text key = new Text();
    Text value = new Text();
    try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path, key.getClass(),
            value.getClass())) {
        key.set("Hello");
        value.set("World");
        writer.append(key, value);
    }
    key.clear();
    value.clear();

    FileStatus status = fs.getFileStatus(path);
    try (InputStream in = new FileInputStream(fs.pathToFile(path));
            SequenceFile.Reader reader = SequenceFileUtil.openReader(in, status, conf)) {
        assertThat(reader.next(key, value), is(true));
        assertThat(key.toString(), is("Hello"));
        assertThat(value.toString(), is("World"));
        assertThat(reader.next(key, value), is(false));
    }
}

From source file:com.asakusafw.runtime.io.sequencefile.SequenceFileUtilTest.java

License:Apache License

/**
 * Reads a sequence file./*from  w w  w .  j a v a  2s .  co  m*/
 * @throws Exception if failed
 */
@Test
public void read_new() throws Exception {
    Path path = new Path("testing");

    Text key = new Text();
    Text value = new Text();
    try (SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path, key.getClass(),
            value.getClass())) {
        key.set("Hello");
        value.set("World");
        writer.append(key, value);
    }
    key.clear();
    value.clear();

    FileStatus status = fs.getFileStatus(path);
    try (InputStream in = new FileInputStream(fs.pathToFile(path));
            SequenceFile.Reader reader = SequenceFileUtil.openReader(in, status.getLen(), conf)) {
        assertThat(reader.next(key, value), is(true));
        assertThat(key.toString(), is("Hello"));
        assertThat(value.toString(), is("World"));
        assertThat(reader.next(key, value), is(false));
    }
}

From source file:com.asakusafw.runtime.io.sequencefile.SequenceFileUtilTest.java

License:Apache License

/**
 * Creates a sequence file.//ww w.j  av  a 2s  . c  om
 * @throws Exception if failed
 */
@Test
public void write() throws Exception {
    Path path = new Path("testing");

    Text key = new Text();
    Text value = new Text();
    try (OutputStream out = new FileOutputStream(fs.pathToFile(path));
            SequenceFile.Writer writer = SequenceFileUtil.openWriter(new BufferedOutputStream(out), conf,
                    key.getClass(), value.getClass(), null)) {
        key.set("Hello");
        value.set("World");
        writer.append(key, value);
    }
    key.clear();
    value.clear();

    try (SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf)) {
        assertThat(reader.next(key, value), is(true));
        assertThat(key.toString(), is("Hello"));
        assertThat(value.toString(), is("World"));
        assertThat(reader.next(key, value), is(false));
    }
}