Example usage for org.apache.hadoop.io Text set

Introduction

In this page you can find the example usage for org.apache.hadoop.io Text set.

Prototype

public void set(Text other) 

Source Link

Document

copy a text.

Usage

From source file:org.apache.accumulo.server.test.functional.BloomFilterTest.java

License:Apache License

private void query(String table, int depth, long start, long end, int num, int step, int secs)
        throws Exception {
    Random r = new Random(42);

    HashSet<Long> expected = new HashSet<Long>();
    List<Range> ranges = new ArrayList<Range>(num);
    Text key = new Text();
    Text row = new Text("row"), cq = new Text("cq"), cf = new Text("cf");

    for (int i = 0; i < num; ++i) {
        Long k = (Math.abs(r.nextLong()) % (end - start)) + start;
        key.set(String.format("k_%010d", k));
        Range range = null;//  w w w  . ja  va2s  .c o m
        Key acuKey;

        if (k % (start + step) == 0) {
            expected.add(k);
        }

        switch (depth) {
        case 1:
            range = new Range(new Text(key));
            break;
        case 2:
            acuKey = new Key(row, key, cq);
            range = new Range(acuKey, true, acuKey.followingKey(PartialKey.ROW_COLFAM), false);
            break;
        case 3:
            acuKey = new Key(row, cf, key);
            range = new Range(acuKey, true, acuKey.followingKey(PartialKey.ROW_COLFAM_COLQUAL), false);
            break;
        }

        ranges.add(range);
    }

    BatchScanner bs = getConnector().createBatchScanner(table, Constants.NO_AUTHS, 3);
    bs.setRanges(ranges);

    long t1 = System.currentTimeMillis();

    for (Entry<Key, Value> entry : bs) {
        long v = Long.parseLong(entry.getValue().toString());
        if (!expected.remove(v)) {
            throw new Exception("Got unexpected return " + entry.getKey() + " " + entry.getValue());
        }
    }

    long t2 = System.currentTimeMillis();

    if (expected.size() > 0) {
        throw new Exception("Did not get all expected values " + expected.size());
    }

    if ((t2 - t1) / 1000.0 >= secs) {
        throw new Exception("Queries exceeded expected run time " + (t2 - t1) / 1000.0 + " " + secs);
    }

}

---

From source file:org.apache.accumulo.server.test.functional.DeleteRowsSplitTest.java

License:Apache License

private void generateRandomRange(Text start, Text end) {
    List<String> bunch = new ArrayList<String>(ROWS);
    Collections.shuffle(bunch);//w ww  .  j a v a  2 s  .c  o  m
    if (bunch.get(0).compareTo((bunch.get(1))) < 0) {
        start.set(bunch.get(0));
        end.set(bunch.get(1));
    } else {
        start.set(bunch.get(1));
        end.set(bunch.get(0));
    }

}

---

From source file:org.apache.accumulo.test.functional.BloomFilterIT.java

License:Apache License

private long query(Connector c, String table, int depth, long start, long end, int num, int step)
        throws Exception {
    Random r = new Random(42);

    HashSet<Long> expected = new HashSet<>();
    List<Range> ranges = new ArrayList<>(num);
    Text key = new Text();
    Text row = new Text("row"), cq = new Text("cq"), cf = new Text("cf");

    for (int i = 0; i < num; ++i) {
        Long k = ((r.nextLong() & 0x7fffffffffffffffl) % (end - start)) + start;
        key.set(String.format("k_%010d", k));
        Range range = null;/*from  w w w  .  j a  v  a 2s  .  c om*/
        Key acuKey;

        if (k % (start + step) == 0) {
            expected.add(k);
        }

        switch (depth) {
        case 1:
            range = new Range(new Text(key));
            break;
        case 2:
            acuKey = new Key(row, key, cq);
            range = new Range(acuKey, true, acuKey.followingKey(PartialKey.ROW_COLFAM), false);
            break;
        case 3:
            acuKey = new Key(row, cf, key);
            range = new Range(acuKey, true, acuKey.followingKey(PartialKey.ROW_COLFAM_COLQUAL), false);
            break;
        }

        ranges.add(range);
    }

    BatchScanner bs = c.createBatchScanner(table, Authorizations.EMPTY, 1);
    bs.setRanges(ranges);

    long t1 = System.currentTimeMillis();
    for (Entry<Key, Value> entry : bs) {
        long v = Long.parseLong(entry.getValue().toString());
        if (!expected.remove(v)) {
            throw new Exception("Got unexpected return " + entry.getKey() + " " + entry.getValue());
        }
    }
    long t2 = System.currentTimeMillis();

    if (expected.size() > 0) {
        throw new Exception("Did not get all expected values " + expected.size());
    }

    bs.close();

    return t2 - t1;
}

---

From source file:org.apache.accumulo.test.functional.BloomFilterTest.java

License:Apache License

private long query(String table, int depth, long start, long end, int num, int step) throws Exception {
    Random r = new Random(42);

    HashSet<Long> expected = new HashSet<Long>();
    List<Range> ranges = new ArrayList<Range>(num);
    Text key = new Text();
    Text row = new Text("row"), cq = new Text("cq"), cf = new Text("cf");

    for (int i = 0; i < num; ++i) {
        Long k = ((r.nextLong() & 0x7fffffffffffffffl) % (end - start)) + start;
        key.set(String.format("k_%010d", k));
        Range range = null;/* w  w w  .ja  v a2  s .  co m*/
        Key acuKey;

        if (k % (start + step) == 0) {
            expected.add(k);
        }

        switch (depth) {
        case 1:
            range = new Range(new Text(key));
            break;
        case 2:
            acuKey = new Key(row, key, cq);
            range = new Range(acuKey, true, acuKey.followingKey(PartialKey.ROW_COLFAM), false);
            break;
        case 3:
            acuKey = new Key(row, cf, key);
            range = new Range(acuKey, true, acuKey.followingKey(PartialKey.ROW_COLFAM_COLQUAL), false);
            break;
        }

        ranges.add(range);
    }

    BatchScanner bs = getConnector().createBatchScanner(table, Constants.NO_AUTHS, 3);
    bs.setRanges(ranges);

    long t1 = System.currentTimeMillis();

    for (Entry<Key, Value> entry : bs) {
        long v = Long.parseLong(entry.getValue().toString());
        if (!expected.remove(v)) {
            throw new Exception("Got unexpected return " + entry.getKey() + " " + entry.getValue());
        }
    }

    long t2 = System.currentTimeMillis();

    if (expected.size() > 0) {
        throw new Exception("Did not get all expected values " + expected.size());
    }

    bs.close();

    return t2 - t1;
}

---

From source file:org.apache.accumulo.test.functional.DeleteRowsSplitIT.java

License:Apache License

private void generateRandomRange(Text start, Text end) {
    List<String> bunch = new ArrayList<>(ROWS);
    Collections.shuffle(bunch);//from  w  ww  .  j a va  2 s  .  co m
    if (bunch.get(0).compareTo((bunch.get(1))) < 0) {
        start.set(bunch.get(0));
        end.set(bunch.get(1));
    } else {
        start.set(bunch.get(1));
        end.set(bunch.get(0));
    }

}

---

From source file:org.apache.accumulo.test.merkle.ingest.RandomWorkload.java

License:Apache License

public void run(final Connector conn, final String tableName, final BatchWriterConfig cfg,
        final long numRecords, int rowMax, int cfMax, int cqMax, int deletePercent) throws Exception {

    final Random rowRand = new Random(12345);
    final Random cfRand = new Random(12346);
    final Random cqRand = new Random(12347);
    final Random deleteRand = new Random(12348);
    long valueCounter = 0l;

    if (!conn.tableOperations().exists(tableName)) {
        conn.tableOperations().create(tableName);
    }/*w  w  w.  j av a 2 s. c  o m*/

    BatchWriter bw = conn.createBatchWriter(tableName, cfg);
    try {
        final Text row = new Text(), cf = new Text(), cq = new Text();
        final Value value = new Value();
        for (long i = 0; i < numRecords; i++) {
            row.set(Integer.toString(rowRand.nextInt(rowMax)));
            cf.set(Integer.toString(cfRand.nextInt(cfMax)));
            cq.set(Integer.toString(cqRand.nextInt(cqMax)));

            Mutation m = new Mutation(row);

            // Choose a random value between [0,100)
            int deleteValue = deleteRand.nextInt(100);

            // putDelete if the value we chose is less than our delete percentage
            if (deleteValue < deletePercent) {
                m.putDelete(cf, cq);
            } else {
                value.set(Long.toString(valueCounter).getBytes());
                m.put(cf, cq, valueCounter, value);
            }

            bw.addMutation(m);

            valueCounter++;
        }
    } finally {
        bw.close();
    }
}

---

From source file:org.apache.accumulo.testing.core.merkle.ingest.RandomWorkload.java

License:Apache License

public void run(final Connector conn, final String tableName, final BatchWriterConfig cfg,
        final long numRecords, int rowMax, int cfMax, int cqMax, int deletePercent) throws Exception {

    final Random rowRand = new Random(12345);
    final Random cfRand = new Random(12346);
    final Random cqRand = new Random(12347);
    final Random deleteRand = new Random(12348);
    long valueCounter = 0l;

    if (!conn.tableOperations().exists(tableName)) {
        conn.tableOperations().create(tableName);
    }//w w  w .j a v a  2  s .com

    BatchWriter bw = conn.createBatchWriter(tableName, cfg);
    try {
        final Text row = new Text(), cf = new Text(), cq = new Text();
        final Value value = new Value();
        for (long i = 0; i < numRecords; i++) {
            row.set(Integer.toString(rowRand.nextInt(rowMax)));
            cf.set(Integer.toString(cfRand.nextInt(cfMax)));
            cq.set(Integer.toString(cqRand.nextInt(cqMax)));

            Mutation m = new Mutation(row);

            // Choose a random value between [0,100)
            int deleteValue = deleteRand.nextInt(100);

            // putDelete if the value we chose is less than our delete
            // percentage
            if (deleteValue < deletePercent) {
                m.putDelete(cf, cq);
            } else {
                value.set(Long.toString(valueCounter).getBytes());
                m.put(cf, cq, valueCounter, value);
            }

            bw.addMutation(m);

            valueCounter++;
        }
    } finally {
        bw.close();
    }
}

---

From source file:org.apache.avro.mapred.TestSequenceFileReader.java

License:Apache License

@BeforeClass
public static void testWriteSequenceFile() throws IOException {
    FILE.delete();/* w  w w.  ja v a  2  s.  co  m*/
    Configuration c = new Configuration();
    URI uri = FILE.toURI();
    SequenceFile.Writer writer = new SequenceFile.Writer(FileSystem.get(uri, c), c, new Path(uri.toString()),
            LongWritable.class, Text.class);
    final LongWritable key = new LongWritable();
    final Text val = new Text();
    for (int i = 0; i < COUNT; ++i) {
        key.set(i);
        val.set(Integer.toString(i));
        writer.append(key, val);
    }
    writer.close();
}

---

From source file:org.apache.camel.component.hdfs.HdfsConsumerTest.java

License:Apache License

@Test
public void testReadString() throws Exception {
    if (!canTest()) {
        return;/*from  w w w .  j  av  a2 s .c o  m*/
    }

    final Path file = new Path(new File("target/test/test-camel-string").getAbsolutePath());
    Configuration conf = new Configuration();
    FileSystem fs1 = FileSystem.get(file.toUri(), conf);
    SequenceFile.Writer writer = createWriter(fs1, conf, file, NullWritable.class, Text.class);
    NullWritable keyWritable = NullWritable.get();
    Text valueWritable = new Text();
    String value = "CIAO!";
    valueWritable.set(value);
    writer.append(keyWritable, valueWritable);
    writer.sync();
    writer.close();

    MockEndpoint resultEndpoint = context.getEndpoint("mock:result", MockEndpoint.class);
    resultEndpoint.expectedMessageCount(1);

    context.addRoutes(new RouteBuilder() {
        public void configure() {
            from("hdfs:///" + file.toUri() + "?fileSystemType=LOCAL&fileType=SEQUENCE_FILE&initialDelay=0")
                    .to("mock:result");
        }
    });
    context.start();

    resultEndpoint.assertIsSatisfied();
}

---

From source file:org.apache.camel.component.hdfs.HdfsConsumerTest.java

License:Apache License

@Test
public void testReadStringArrayFile() throws Exception {
    if (!canTest()) {
        return;/*from   w  w  w.  j av  a 2  s  .  co m*/
    }

    final Path file = new Path(new File("target/test/test-camel-string").getAbsolutePath());
    Configuration conf = new Configuration();
    FileSystem fs1 = FileSystem.get(file.toUri(), conf);
    ArrayFile.Writer writer = new ArrayFile.Writer(conf, fs1, "target/test/test-camel-string1", Text.class,
            CompressionType.NONE, new Progressable() {
                @Override
                public void progress() {
                }
            });
    Text valueWritable = new Text();
    String value = "CIAO!";
    valueWritable.set(value);
    writer.append(valueWritable);
    writer.close();

    MockEndpoint resultEndpoint = context.getEndpoint("mock:result", MockEndpoint.class);
    resultEndpoint.expectedMessageCount(1);

    context.addRoutes(new RouteBuilder() {
        public void configure() {
            from("hdfs:///" + file.getParent().toUri()
                    + "?fileSystemType=LOCAL&fileType=ARRAY_FILE&initialDelay=0").to("mock:result");
        }
    });
    context.start();

    resultEndpoint.assertIsSatisfied();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• »