Example usage for org.apache.hadoop.io Text set

Introduction

In this page you can find the example usage for org.apache.hadoop.io Text set.

Prototype

public void set(Text other)

Source Link

Document

copy a text.

Usage

From source file:com.facebook.presto.accumulo.serializers.StringRowSerializer.java

License:Apache License

@Override
public void setInt(Text text, Integer value) {
    text.set(value.toString().getBytes(UTF_8));
}

From source file:com.facebook.presto.accumulo.serializers.StringRowSerializer.java

License:Apache License

@Override
public void setLong(Text text, Long value) {
    text.set(value.toString().getBytes(UTF_8));
}

From source file:com.facebook.presto.accumulo.serializers.StringRowSerializer.java

License:Apache License

@Override
public void setShort(Text text, Short value) {
    text.set(value.toString().getBytes(UTF_8));
}

From source file:com.facebook.presto.accumulo.serializers.StringRowSerializer.java

License:Apache License

@Override
public void setTime(Text text, Time value) {
    text.set(Long.toString(value.getTime()).getBytes(UTF_8));
}

From source file:com.facebook.presto.accumulo.serializers.StringRowSerializer.java

License:Apache License

@Override
public void setTimestamp(Text text, Timestamp value) {
    text.set(Long.toString(value.getTime()).getBytes(UTF_8));
}

From source file:com.facebook.presto.accumulo.serializers.StringRowSerializer.java

License:Apache License

@Override
public void setVarbinary(Text text, byte[] value) {
    text.set(value);
}

From source file:com.facebook.presto.accumulo.serializers.StringRowSerializer.java

License:Apache License

@Override
public void setVarchar(Text text, String value) {
    text.set(value.getBytes(UTF_8));
}

From source file:com.facebook.presto.accumulo.tools.RewriteIndex.java

License:Apache License

private void addIndexEntries(Connector connector, AccumuloTable table, long start) {
    LOG.info(format("Scanning data table %s to add index entries", table.getFullTableName()));
    BatchScanner scanner = null;//w w w.j a v  a 2  s  .  c om
    BatchWriter indexWriter = null;
    try {
        // Create index writer and metrics writer, but we are never going to flush the metrics writer
        indexWriter = connector.createBatchWriter(table.getIndexTableName(), bwc);
        Indexer indexer = new Indexer(connector, table, indexWriter,
                table.getMetricsStorageInstance(connector).newWriter(table));
        LOG.info("Created indexer against " + table.getIndexTableName());

        scanner = connector.createBatchScanner(table.getFullTableName(), auths, 10);
        LOG.info(format("Created batch scanner against %s with auths %s", table.getFullTableName(), auths));

        IteratorSetting timestampFilter = new IteratorSetting(21, "timestamp", TimestampFilter.class);
        TimestampFilter.setRange(timestampFilter, 0L, start);
        scanner.addScanIterator(timestampFilter);

        scanner.setRanges(connector.tableOperations().splitRangeByTablets(table.getFullTableName(), new Range(),
                Integer.MAX_VALUE));

        long numRows = 0L;
        long numIndexEntries = 0L;
        Text prevRow = null;
        Text row = new Text();
        Text cf = new Text();
        Text cq = new Text();
        Mutation mutation = null;
        for (Entry<Key, Value> entry : scanner) {
            entry.getKey().getRow(row);
            entry.getKey().getColumnFamily(cf);
            entry.getKey().getColumnQualifier(cq);

            // if the rows do not match, index the mutation
            if (prevRow != null && !prevRow.equals(row)) {
                if (!dryRun) {
                    indexer.index(mutation);
                }
                ++numRows;
                mutation = null;

                if (numRows % 500000 == 0) {
                    if (dryRun) {
                        LOG.info(
                                format("In progress, would have re-indexed %s rows containing %s index entries",
                                        numRows, numIndexEntries));
                    } else {
                        LOG.info(format("In progress, re-indexed %s rows containing %s index entries", numRows,
                                numIndexEntries));
                    }
                }
            }

            if (mutation == null) {
                mutation = new Mutation(row);
            }

            mutation.put(cf, cq, entry.getKey().getColumnVisibilityParsed(), entry.getKey().getTimestamp(),
                    entry.getValue());
            if (table.getColumns().stream()
                    .filter(column -> column.isIndexed() && column.getFamily().isPresent()
                            && column.getQualifier().isPresent()
                            && column.getFamily().get().equals(new String(cf.copyBytes(), UTF_8))
                            && column.getQualifier().get().equals(new String(cq.copyBytes(), UTF_8)))
                    .count() > 0) {
                ++numIndexEntries;
            }

            if (prevRow == null) {
                prevRow = new Text(row);
            } else {
                prevRow.set(row);
            }
        }

        // Index the final mutation
        if (mutation != null) {
            if (!dryRun) {
                indexer.index(mutation);
            }
            ++numRows;
        }

        if (dryRun) {
            LOG.info(format(
                    "Finished dry run of rewriting index entries. Would have re-indexed %s rows containing %s index entries",
                    numRows, numIndexEntries));
        } else {
            LOG.info(format("Finished adding index entries. Re-indexed %s rows containing %s index entries",
                    numRows, numIndexEntries));
        }
    } catch (AccumuloException | AccumuloSecurityException e) {
        LOG.error("Accumulo exception", e);
    } catch (TableNotFoundException e) {
        LOG.error("Table not found, must have been deleted during process", e);
    } finally {
        if (indexWriter != null) {
            try {
                indexWriter.close();
            } catch (MutationsRejectedException e) {
                LOG.error("Server rejected mutations", e);
            }
        }

        if (scanner != null) {
            scanner.close();
        }
    }
}

From source file:com.facebook.presto.accumulo.tools.RewriteMetricsTask.java

License:Apache License

private void rewriteMetrics(Connector connector, AccumuloTable table, long start) {
    LOG.info("Rewriting metrics for table " + table.getFullTableName());

    TypedValueCombiner.Encoder<Long> encoder = new LongCombiner.StringEncoder();
    BatchWriter writer = null;/*from w ww .j av  a2 s.  c  o m*/
    Scanner scanner = null;
    try {
        writer = connector.createBatchWriter(table.getIndexTableName() + "_metrics", bwc);
        LOG.info("Created batch writer against " + table.getIndexTableName() + "_metrics");

        scanner = new IsolatedScanner(connector.createScanner(table.getIndexTableName(), auths));
        LOG.info(format("Created isolated scanner against %s with auths %s", table.getIndexTableName(), auths));

        Set<Pair<String, String>> timestampColumns = table.isTruncateTimestamps() ? table.getColumns().stream()
                .filter(x -> x.getType().equals(TimestampType.TIMESTAMP) && x.getFamily().isPresent())
                .map(x -> Pair.of(x.getFamily().get(), x.getQualifier().get())).collect(Collectors.toSet())
                : ImmutableSet.of();

        LOG.info("Timestamp columns are " + timestampColumns);

        IteratorSetting timestampFilter = new IteratorSetting(21, "timestamp", TimestampFilter.class);
        TimestampFilter.setRange(timestampFilter, 0L, start);
        scanner.addScanIterator(timestampFilter);

        Map<Text, Map<Text, Map<ColumnVisibility, AtomicLong>>> rowMap = new HashMap<>();
        long numMutations = 0L;
        boolean warned = true;
        Text prevRow = null;
        for (Entry<Key, Value> entry : scanner) {
            Text row = entry.getKey().getRow();
            Text cf = entry.getKey().getColumnFamily();

            if (prevRow != null && !prevRow.equals(row)) {
                writeMetrics(start, encoder, writer, rowMap);
                ++numMutations;

                if (numMutations % 500000 == 0) {
                    if (dryRun) {
                        LOG.info(format("In progress, would have written %s metric mutations", numMutations));
                    } else {
                        LOG.info("In progress, metric mutations written: " + numMutations);
                    }
                }
            }

            ColumnVisibility visibility = entry.getKey().getColumnVisibilityParsed();
            incrementMetric(rowMap, row, cf, visibility);
            String[] famQual = cf.toString().split("_");

            if (famQual.length == 2) {
                if (timestampColumns.contains(Pair.of(famQual[0], famQual[1]))) {
                    incrementTimestampMetric(rowMap, cf, visibility, row);
                }
            } else if (warned) {
                LOG.warn(
                        "Unable to re-write timestamp metric when either of a family/qualifier column mapping contains an underscore");
                warned = false;
            }

            if (prevRow == null) {
                prevRow = new Text(row);
            } else {
                prevRow.set(row);
            }
        }

        // Write final metric
        writeMetrics(start, encoder, writer, rowMap);
        ++numMutations;

        if (dryRun) {
            LOG.info(format("Would have written %s mutations", numMutations));
        } else {
            LOG.info("Finished rewriting metrics. Mutations written: " + numMutations);
        }
    } catch (TableNotFoundException e) {
        LOG.error("Table not found, must have been deleted during process", e);
    } catch (MutationsRejectedException e) {
        LOG.error("Server rejected mutations", e);
    } finally {
        if (writer != null) {
            try {
                writer.close();
            } catch (MutationsRejectedException e) {
                LOG.error("Server rejected mutations", e);
            }
        }

        if (scanner != null) {
            scanner.close();
        }
    }
}

From source file:com.foobar.store.ToJSONSeqConverter.java

License:Apache License

/**
 * @param i/*from  w ww.j  ava  2s.c o m*/
 * @param t
 */
private void convertItemToText(JsonValue val, Text t) {
    if (val == null || t == null)
        return;

    try {
        String s = convertItemToString(val);
        t.set(s.getBytes());
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}