List of usage examples for com.amazonaws.services.kinesis.model Record getData
public java.nio.ByteBuffer getData()
The data blob.
From source file:com.datatorrent.contrib.kinesis.KinesisStringInputOperator.java
License:Open Source License
/** * Implement abstract method of AbstractKinesisInputOperator *///w w w . j a va2 s . c om @Override public String getTuple(Record record) { try { ByteBuffer bb = record.getData(); byte[] bytes = new byte[bb.remaining()]; bb.get(bytes); return new String(bytes); } catch (Exception ex) { throw new RuntimeException(ex); } }
From source file:com.datatorrent.contrib.kinesis.KinesisTestConsumer.java
License:Open Source License
public String getData(Record rc) { ByteBuffer buffer = rc.getData(); byte[] bytes = new byte[buffer.remaining()]; buffer.get(bytes);//from w w w . j a va 2 s. com return new String(bytes); }
From source file:com.github.jramos.snowplow.RedshiftSinkModelTransformer.java
License:Apache License
@Override public SnowplowEventModel toClass(Record record) throws IOException { SnowplowEventModel event = null;/*w w w. j a va2 s. c o m*/ String dataRow = null; try { dataRow = new String(record.getData().array(), "UTF-8"); event = handler.process(dataRow); if (eventAppLogList != null) { if (eventAppLogList.contains(event.getApp_id())) { LOG.info(dataRow); } } if (operators != null) { for (ISnowplowEventOperator operator : operators) { event = operator.apply(event); } } } catch (IOException ioe) { LOG.error("toClass. Processing dataRow " + dataRow, ioe); throw ioe; } return event; }
From source file:com.hortonworks.streamline.streams.runtime.storm.spout.KinesisRecordToTupleMapper.java
License:Apache License
@Override public List<Object> getTuple(Record record) { CharsetDecoder decoder = Charset.forName("UTF-8").newDecoder(); List<Object> tuple = new ArrayList<>(); tuple.add(record.getPartitionKey()); tuple.add(record.getSequenceNumber()); try {/*w ww. j a va2s . c o m*/ String data = decoder.decode(record.getData()).toString(); tuple.add(data); } catch (CharacterCodingException e) { e.printStackTrace(); LOG.warn("Exception occured. Emitting tuple with empty string data", e); tuple.add(""); } return tuple; }
From source file:com.importio.kinesis.elasticsearch.KinesisTransformer.java
License:Open Source License
@Override public Collection<Page> toClass(Record record) throws IOException { List<Page> list = Lists.newArrayList(); BufferedReader reader = new BufferedReader( new InputStreamReader(new ByteArrayInputStream(record.getData().array()))); String line;//w w w. j av a 2 s. c o m while ((line = reader.readLine()) != null) { try { Page page = mapper.readValue(line, Page.class); if (!EVENT_TYPE.equals(page.getMetaEventType())) { continue; } if (page.getMetaIp() != null && !page.getMetaIp().equals("127.0.0.1")) { setGeoFields(page); } list.add(page); } catch (Exception e) { log.error("Failed conversion: " + line, e); } } return list; }
From source file:com.informatica.surf.sample.RecordProcessor.java
License:Apache License
@Override public void processRecords(List<Record> list, IRecordProcessorCheckpointer irpc) { _logger.info("Processing {} records", list.size()); for (Record r : list) { String data = new String(r.getData().array()); long seq = _buffer.next(); KinesisEvent evt = _buffer.get(seq); evt.setData(data);//from ww w . ja v a2 s. c om _buffer.publish(seq); } try { irpc.checkpoint(); } catch (InvalidStateException | KinesisClientLibDependencyException | ShutdownException | ThrottlingException ex) { _logger.warn("Exception while checkpointing", ex); } }
From source file:com.kinesis.datavis.kcl.processor.CountingRecordProcessor.java
License:Open Source License
@Override public void processRecords(List<Record> records, IRecordProcessorCheckpointer checkpointer) { for (Record r : records) { // Deserialize each record as an UTF-8 encoded JSON String of the type provided T rec;//from w w w . ja v a2s.com try { rec = JSON.readValue(r.getData().array(), recordType); rec = typeProcessor.process(r, rec); } catch (IOException e) { LOG.warn("Skipping record. Unable to parse record into HttpReferrerPair. Partition Key: " + r.getPartitionKey() + ". Sequence Number: " + r.getSequenceNumber(), e); continue; } synchronized (windowCounter) { windowCounter.increment(rec); } } if (buffer.shouldFlush()) { emit(buffer.getRecords()); } // Checkpoint if it's time to! if (checkpointTimer.isTimeUp()) { // Obtain a lock on the windowCounter to prevent additional counts from being calculated while checkpointing. synchronized (windowCounter) { checkpoint(checkpointer); resetCheckpointAlarm(); } } }
From source file:com.kinesis.processor.StockTradeRecordProcessor.java
License:Open Source License
private void processRecord(Record record) { StockTrade trade = StockTrade.fromJsonAsBytes(record.getData().array()); if (trade == null) { LOG.warn("Skipping record. Unable to parse record into StockTrade. Partition Key: " + record.getPartitionKey()); return;//ww w .ja v a2 s . c o m } stockStats.addStockTrade(trade); }
From source file:com.netflix.spectator.tdigest.KinesisTDigestReader.java
License:Apache License
@Override public List<TDigestMeasurement> read() throws IOException { init();//from w w w .ja va 2 s . c o m if (recRequest.getShardIterator() == null) { return Collections.emptyList(); } else { List<TDigestMeasurement> ms = new ArrayList<>(); GetRecordsResult result = client.getRecords(recRequest); recRequest.setShardIterator(result.getNextShardIterator()); for (Record r : result.getRecords()) { recordsProcessed.increment(); ByteBuffer data = r.getData(); try { ms.addAll(json.decode(data.array())); } catch (Exception e) { recordsSkipped.increment(); LOGGER.warn("failed to decode record, skipping (" + iterRequest + ")", e); } } return ms; } }
From source file:com.nodestone.ksum.RecordProcessor.java
License:Open Source License
/** * Process a single record./*from ww w . j a v a 2 s . com*/ * * @param record The record to be processed. */ private void processSingleRecord(Record record) { // TODO Add your own record processing logic here String data = null; RawData rawData = null; try { // For this app, we interpret the payload as UTF-8 chars. data = decoder.decode(record.getData()).toString(); try { rawData = null; rawData = RawData.fromJSON(data); LOG.info("raw: " + rawData.custId.toString() + " " + rawData.value.toString()); } catch (Exception e) { LOG.info("Can't decode raw data as json, ignoring - data was:" + data); } if (rawData != null) { this.updateRecord(rawData); } LOG.info(record.getSequenceNumber() + ", " + record.getPartitionKey() + ", " + data); } catch (NumberFormatException e) { LOG.info("Record does not match sample record format. Ignoring record with data; " + data); } catch (CharacterCodingException e) { LOG.error("Malformed data: " + data, e); } }