List of usage examples for io.netty.buffer ByteBuf getUnsignedByte
public abstract short getUnsignedByte(int index);
From source file:com.github.sadikovi.netflowlib.NetFlowReader.java
License:Apache License
/** * [[NetFlowReader]] provides interface to get parsed header and record buffer with chosen * strategy based on columns, predicate and statistics. Metadata, header are parsed as part of * initialization./*from ww w .j av a2 s .c o m*/ */ private NetFlowReader(DataInputStream inputStream, int buffer, boolean ignoreCorruptFile) throws IOException { in = inputStream; bufferLength = buffer; ignoreCorrupt = ignoreCorruptFile; byte[] metadata = null; ByteBuf buf = null; try { metadata = new byte[METADATA_LENGTH]; in.read(metadata, 0, METADATA_LENGTH); // Parse metadata, byte order does not really matter, so we go for big endian. Metadata contains // magic numbers to verify consistency of the NetFlow file, byte order encoded as either 1 or 2, // and stream version which affects header parsing (currently only 1 and 3 are supported). buf = Unpooled.wrappedBuffer(metadata).order(ByteOrder.BIG_ENDIAN); short magic1 = buf.getUnsignedByte(0); short magic2 = buf.getUnsignedByte(1); short order = buf.getUnsignedByte(2); short stream = buf.getUnsignedByte(3); // Verify consistency of NetFlow file, also this ensures that we are at the beginning of the // input stream if (magic1 != HEADER_MAGIC1 || magic2 != HEADER_MAGIC2) { throw new IOException("Corrupt NetFlow file. Wrong magic number"); } // Resolve byte order, last case corresponds to incorrect reading from buffer if (order == HEADER_BIG_ENDIAN) { byteOrder = ByteOrder.BIG_ENDIAN; } else if (order == HEADER_LITTLE_ENDIAN) { byteOrder = ByteOrder.LITTLE_ENDIAN; } else { throw new IOException("Could not recognize byte order " + order); } streamVersion = stream; // Check stream version ensureStreamVersion(); // Read header header = getHeader(); } catch (IOException err) { if (ignoreCorrupt) { // we subsume exception and log warning. Set header to null log.warn("Failed to initialize reader, ignoreCorruptFile=" + ignoreCorrupt + ", error=" + err); header = new CorruptNetFlowHeader(); } else { throw err; } } finally { metadata = null; if (buf != null) { buf.release(); buf = null; } } }
From source file:com.github.sadikovi.netflowlib.NetFlowReader.java
License:Apache License
/** Prepare header using provided input stream */ private NetFlowHeader prepareHeader() throws IOException { NetFlowHeader internalHeader;//from w w w . ja v a2 s . com int numBytesRead = 0; int lenRead = 0; ByteBuf buf; byte[] headerArray; // Read header depending on stream version (different from flow version) if (streamVersion == 1) { // Version 1 has static header // TODO: verify header size for stream version 1 lenRead = NetFlowHeader.S1_HEADER_SIZE - METADATA_LENGTH; internalHeader = new NetFlowHeader(streamVersion, byteOrder); } else { // Version 3 with dynamic header size headerArray = new byte[HEADER_OFFSET_LENGTH]; numBytesRead = in.read(headerArray, 0, HEADER_OFFSET_LENGTH); if (numBytesRead != HEADER_OFFSET_LENGTH) { throw new UnsupportedOperationException("Short read while loading header offset"); } buf = Unpooled.wrappedBuffer(headerArray).order(byteOrder); int headerSize = (int) buf.getUnsignedInt(0); if (headerSize <= 0) { throw new UnsupportedOperationException("Failed to load header of size " + headerSize); } // Actual header length, determine how many bytes to read lenRead = headerSize - METADATA_LENGTH - HEADER_OFFSET_LENGTH; internalHeader = new NetFlowHeader(streamVersion, byteOrder, headerSize); } // allocate buffer for length to read headerArray = new byte[lenRead]; numBytesRead = in.read(headerArray, 0, lenRead); if (numBytesRead != lenRead) { throw new UnsupportedOperationException("Short read while loading header data"); } // build buffer buf = Unpooled.wrappedBuffer(headerArray).order(byteOrder); // resolve stream version (either 1 or 3) if (streamVersion == 1) { internalHeader.setFlowVersion((short) buf.getUnsignedShort(0)); internalHeader.setStartCapture(buf.getUnsignedInt(2)); internalHeader.setEndCapture(buf.getUnsignedInt(6)); internalHeader.setHeaderFlags(buf.getUnsignedInt(10)); internalHeader.setRotation(buf.getUnsignedInt(14)); internalHeader.setNumFlows(buf.getUnsignedInt(18)); internalHeader.setNumDropped(buf.getUnsignedInt(22)); internalHeader.setNumMisordered(buf.getUnsignedInt(26)); // Read hostname fixed bytes byte[] hostnameBytes = new byte[NetFlowHeader.S1_HEADER_HN_LEN]; buf.getBytes(30, hostnameBytes, 0, hostnameBytes.length); internalHeader.setHostname(new String(hostnameBytes)); // Read comments fixed bytes byte[] commentsBytes = new byte[NetFlowHeader.S1_HEADER_CMNT_LEN]; buf.getBytes(30 + hostnameBytes.length, commentsBytes, 0, commentsBytes.length); internalHeader.setComments(new String(commentsBytes)); // Dereference arrays hostnameBytes = null; commentsBytes = null; } else { // Resolve TLV (type-length value) // Set decode pointer to first tlv int dp = 0; int left = lenRead; // Smallest TLV is 2+2+0 (null TLV) // tlv_t - TLV type, tlv_l - TLV length, tlv_v - TLV value int tlv_t = 0; int tlv_l = 0; int tlv_v = 0; // Byte array for holding Strings byte[] pr; while (left >= 4) { // Parse type, store in host byte order tlv_t = buf.getUnsignedShort(dp); dp += 2; left -= 2; // Parse len, store in host byte order tlv_l = buf.getUnsignedShort(dp); dp += 2; left -= 2; // Parse val tlv_v = dp; // Point decode buffer at next tlv dp += tlv_l; left -= tlv_l; // TLV length check if (left < 0) { break; } switch (tlv_t) { // FT_TLV_VENDOR case 0x1: internalHeader.setVendor(buf.getUnsignedShort(tlv_v)); break; // FT_TLV_EX_VER case 0x2: internalHeader.setFlowVersion((short) buf.getUnsignedShort(tlv_v)); break; // FT_TLV_AGG_VER case 0x3: internalHeader.setAggVersion(buf.getUnsignedByte(tlv_v)); break; // FT_TLV_AGG_METHOD case 0x4: internalHeader.setAggMethod(buf.getUnsignedByte(tlv_v)); break; // FT_TLV_EXPORTER_IP case 0x5: internalHeader.setExporterIP(buf.getUnsignedInt(tlv_v)); break; // FT_TLV_CAP_START case 0x6: internalHeader.setStartCapture(buf.getUnsignedInt(tlv_v)); break; // FT_TLV_CAP_END case 0x7: internalHeader.setEndCapture(buf.getUnsignedInt(tlv_v)); break; // FT_TLV_HEADER_FLAGS case 0x8: internalHeader.setHeaderFlags(buf.getUnsignedInt(tlv_v)); break; // FT_TLV_ROT_SCHEDULE case 0x9: internalHeader.setRotation(buf.getUnsignedInt(tlv_v)); break; // FT_TLV_FLOW_COUNT case 0xA: internalHeader.setNumFlows(buf.getUnsignedInt(tlv_v)); break; // FT_TLV_FLOW_LOST case 0xB: internalHeader.setNumDropped(buf.getUnsignedInt(tlv_v)); break; // FT_TLV_FLOW_MISORDERED case 0xC: internalHeader.setNumMisordered(buf.getUnsignedInt(tlv_v)); break; // FT_TLV_PKT_CORRUPT case 0xD: internalHeader.setNumCorrupt(buf.getUnsignedInt(tlv_v)); break; // FT_TLV_SEQ_RESET case 0xE: internalHeader.setSeqReset(buf.getUnsignedInt(tlv_v)); break; // FT_TLV_CAP_HOSTNAME case 0xF: pr = new byte[tlv_l]; buf.getBytes(tlv_v, pr, 0, pr.length); // Expected null-terminated string if (pr[pr.length - 1] != 0) { throw new UnsupportedOperationException("Char sequence is not null-terminated"); } internalHeader.setHostname(new String(pr, 0, pr.length - 1)); break; // FT_TLV_COMMENTS case 0x10: pr = new byte[tlv_l]; buf.getBytes(tlv_v, pr, 0, pr.length); // Expected null-terminated string if (pr[pr.length - 1] != 0) { throw new UnsupportedOperationException("Char sequence is not null-terminated"); } internalHeader.setComments(new String(pr, 0, pr.length - 1)); break; // FT_TLV_IF_NAME case 0x11: // uint32_t, uint16_t, string: // - IP address of device // - ifIndex of interface // - interface name long ip = buf.getUnsignedInt(tlv_v); int ifIndex = buf.getUnsignedShort(tlv_v + 4); pr = new byte[tlv_l - 4 - 2]; buf.getBytes(tlv_v + 4 + 2, pr, 0, pr.length); if (pr[pr.length - 1] != 0) { throw new UnsupportedOperationException("Char sequence is not null-terminated"); } internalHeader.setInterfaceName(ip, ifIndex, new String(pr, 0, pr.length - 1)); break; // FT_TLV_IF_ALIAS case 0x12: // uint32_t, uint16_t, uint16_t, string: // - IP address of device // - ifIndex count // - ifIndex of interface (count times) // - alias name long aliasIP = buf.getUnsignedInt(tlv_v); int aliasIfIndexCnt = buf.getUnsignedShort(tlv_v + 4); int aliasIfIndex = buf.getUnsignedShort(tlv_v + 4 + 2); pr = new byte[tlv_l - 4 - 2 - 2]; buf.getBytes(tlv_v + 4 + 2 + 2, pr, 0, pr.length); if (pr[pr.length - 1] != 0) { throw new UnsupportedOperationException("Char sequence is not null-terminated"); } internalHeader.setInterfaceAlias(aliasIP, aliasIfIndexCnt, aliasIfIndex, new String(pr, 0, pr.length - 1)); break; // Case 0x0 default: break; } } if (buf != null && buf.refCnt() > 0) { buf.release(buf.refCnt()); } buf = null; pr = null; } return internalHeader; }
From source file:com.github.sadikovi.netflowlib.record.RecordMaterializer.java
License:Apache License
/** Read buffer bytes sequence for column offset */ public Object readField(Column column, ByteBuf buffer) { Class<?> type = column.getColumnType(); if (type.equals(Byte.class)) { return buffer.getByte(column.getColumnOffset()); } else if (type.equals(Short.class)) { return buffer.getUnsignedByte(column.getColumnOffset()); } else if (type.equals(Integer.class)) { return buffer.getUnsignedShort(column.getColumnOffset()); } else if (type.equals(Long.class)) { return buffer.getUnsignedInt(column.getColumnOffset()); } else {/*from w ww.j av a 2 s . co m*/ throw new UnsupportedOperationException("Unsupported read type " + type); } }
From source file:com.github.sadikovi.netflowlib.record.RecordMaterializer.java
License:Apache License
public void updateValueInspector(Column column, ByteBuf buffer, ValueInspector vi) { Class<?> type = column.getColumnType(); if (type.equals(Byte.class)) { vi.update(buffer.getByte(column.getColumnOffset())); } else if (type.equals(Short.class)) { vi.update(buffer.getUnsignedByte(column.getColumnOffset())); } else if (type.equals(Integer.class)) { vi.update(buffer.getUnsignedShort(column.getColumnOffset())); } else if (type.equals(Long.class)) { vi.update(buffer.getUnsignedInt(column.getColumnOffset())); } else {//from w ww . j a va2s . c o m throw new UnsupportedOperationException("Unsupported read type " + type); } }
From source file:com.heliosapm.streams.onramp.GZipDetector.java
License:Apache License
/** * {@inheritDoc}//w w w . ja va 2 s . c o m * @see io.netty.handler.codec.ByteToMessageDecoder#decode(io.netty.channel.ChannelHandlerContext, io.netty.buffer.ByteBuf, java.util.List) */ @Override protected void decode(final ChannelHandlerContext ctx, final ByteBuf buff, final List<Object> out) throws Exception { if (buff.readableBytes() > 4) { final int magic1 = buff.getUnsignedByte(buff.readerIndex()); final int magic2 = buff.getUnsignedByte(buff.readerIndex() + 1); if (isGzip(magic1, magic2)) { enableGzip(ctx); } else { ctx.pipeline().remove(this); } out.add(buff.retain()); } }
From source file:com.heliosapm.tsdblite.handlers.ProtocolSwitch.java
License:Apache License
/** * {@inheritDoc}/*ww w.j a v a 2s . c o m*/ * @see io.netty.handler.codec.ByteToMessageDecoder#decode(io.netty.channel.ChannelHandlerContext, io.netty.buffer.ByteBuf, java.util.List) */ @Override protected void decode(final ChannelHandlerContext ctx, final ByteBuf in, final List<Object> out) throws Exception { // Will use the first five bytes to detect a protocol. if (in.readableBytes() < 5) { log.info("No ProtocolSwitch. Bytes: {}", in.readableBytes()); return; } final int magic1 = in.getUnsignedByte(in.readerIndex()); final int magic2 = in.getUnsignedByte(in.readerIndex() + 1); if (detectGzip && isGzip(magic1, magic2)) { enableGzip(ctx); log.info("Enabled GZip on channel [{}]", ctx.channel().id().asShortText()); } else if (isHttp(magic1, magic2)) { switchToHttp(ctx); log.info("Switched to HTTP on channel [{}]", ctx.channel().id().asShortText()); } else if (isText(magic1, magic2)) { switchToPlainText(ctx); log.info("Switched to PlainText on channel [{}]", ctx.channel().id().asShortText()); } else { log.error("No protocol recognized on [{}]", ctx.channel().id().asLongText()); in.clear(); ctx.close(); } }
From source file:com.hzmsc.scada.Jmtis.server.PortUnificationServerHandler.java
License:Apache License
@Override protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception { System.out.println(in.getUnsignedByte(in.readerIndex()) + ".........................."); // Will use the first five bytes to detect a protocol. if (in.readableBytes() < 5) { return;/*from w w w.ja va 2 s.co m*/ } if (isSsl(in)) { enableSsl(ctx); } else { final int magic1 = in.getUnsignedByte(in.readerIndex()); final int magic2 = in.getUnsignedByte(in.readerIndex() + 1); if (isGzip(magic1, magic2)) { enableGzip(ctx); } else if (isHttp(magic1, magic2)) { switchToHttp(ctx); } else if (isJMAC(magic1, magic2)) { switchToJMAC(ctx); } else { // Unknown protocol; discard everything and close the connection. in.clear(); ctx.close(); } } }
From source file:com.netty.test.LengthFieldBasedExFrameDecoder.java
License:Apache License
private long getFrameLength(ByteBuf in, int actualLengthFieldOffset) { in = in.order(byteOrder);//www .j a v a 2s . c om long frameLength; switch (lengthFieldLength) { case 1: frameLength = in.getUnsignedByte(actualLengthFieldOffset); break; case 2: frameLength = in.getUnsignedShort(actualLengthFieldOffset); break; case 3: frameLength = in.getUnsignedMedium(actualLengthFieldOffset); break; case 4: frameLength = in.getUnsignedInt(actualLengthFieldOffset); break; case 8: frameLength = in.getLong(actualLengthFieldOffset); break; default: throw new Error("should not reach here"); } return frameLength; }
From source file:com.shelf.messagepack.MessagePackFrameDecoder.java
License:Apache License
public long decodeLength(ByteBuf in, int offset) throws Exception { if (discardingTooLongFrame) { long bytesToDiscard = this.bytesToDiscard; int localBytesToDiscard = (int) Math.min(bytesToDiscard, in.readableBytes()); in.skipBytes(localBytesToDiscard); bytesToDiscard -= localBytesToDiscard; this.bytesToDiscard = bytesToDiscard; failIfNecessary(false);// w w w . j av a 2 s. co m } int readerIndex = in.readerIndex() + offset; short b = in.getUnsignedByte(readerIndex); int ubyte = b & 0xff; LOGGER.trace("message: " + toHex(ubyte)); switch (ubyte) { case NIL: return 1L; case FALSE: return 1L; case TRUE: return 1L; case BIN8: { short length = in.getUnsignedByte(readerIndex + 1); return 2L + length; } case BIN16: { int length = in.getUnsignedShort(readerIndex + 1); return 3L + length; } case BIN32: { long length = in.getUnsignedInt(readerIndex + 1); return 5L + length; } case EXT8: { short length = in.getUnsignedByte(readerIndex + 1); return 3L + length; } case EXT16: { int length = in.getUnsignedShort(readerIndex + 1); return 4L + length; } case EXT32: { long length = in.getUnsignedInt(readerIndex + 1); return 6L + length; } case FLOAT32: return 5L; case FLOAT64: return 9L; case UINT8: return 2L; case UINT16: return 3L; case UINT32: return 5L; case UINT64: return 9L; case INT8: return 2L; case INT16: return 3L; case INT32: return 5L; case INT64: return 9L; case FIXEXT1: return 3L; case FIXEXT2: return 4L; case FIXEXT4: return 6L; case FIXEXT8: return 10L; case FIXEXT16: return 18L; case STR8: { short length = in.getUnsignedByte(readerIndex + 1); return 2L + length; } case STR16: { int length = in.getUnsignedShort(readerIndex + 1); return 3L + length; } case STR32: { long length = in.getUnsignedInt(readerIndex + 1); return 5L + length; } case ARRAY16: { int elemCount = in.getUnsignedShort(readerIndex + 1); return getArraySize(in, 3, offset, elemCount); } case ARRAY32: { long elemCount = in.getUnsignedInt(readerIndex + 1); return getArraySize(in, 5, offset, elemCount); } case MAP16: { int elemCount = in.getUnsignedShort(readerIndex + 1); return getArraySize(in, 3, offset, elemCount * 2); } case MAP32: { long elemCount = in.getUnsignedInt(readerIndex + 1); return getArraySize(in, 5, offset, elemCount * 2); } default: if ((ubyte >> 7) == 0) { //positive fixint return 1L; } else if ((ubyte >> 4) == 0b1000) { //fixmap int elemCount = ubyte & 0b00001111; return getArraySize(in, 1, offset, elemCount * 2); } else if ((ubyte >> 4) == 0b1001) { //fixarray int elemCount = ubyte & 0b00001111; return getArraySize(in, 1, offset, elemCount); } else if ((ubyte >> 5) == 0b101) { //fixstr int length = ubyte & 0b00011111; return 1L + length; } else if ((ubyte >> 5) == 0b111) { //negative fixint return 1L; } else { throw new CorruptedFrameException("Unknown header byte of message: " + toHex(ubyte)); } } }
From source file:com.streamsets.pipeline.lib.parser.netflow.NetflowParser.java
License:Apache License
private List<Record> parseV5(int version, int packetLength, ByteBuf buf, String readerId, InetSocketAddress sender) throws OnRecordErrorException { int count = buf.getUnsignedShort(2); // 2-3 if (count <= 0) { throw new OnRecordErrorException(Errors.NETFLOW_01, Utils.format("Count is invalid: {}", count)); } else if (packetLength < V5_HEADER_SIZE + count * V5_FLOW_SIZE) { String msg = Utils.format("Readable bytes {} too small for count {} (max {})", packetLength, count, (V5_HEADER_SIZE + count * V5_FLOW_SIZE)); throw new OnRecordErrorException(Errors.NETFLOW_01, msg); }//ww w . jav a 2 s . c o m List<Record> result = new ArrayList<>(count); long uptime = buf.getUnsignedInt(4); // 4-7 long seconds = buf.getUnsignedInt(8); // 8-11 long millis = buf.getUnsignedInt(8) / 1000; // 12-15 long timestamp = (seconds * 1000L) + millis; // java timestamp, which is milliseconds UUID packetId = UUIDs.startOfJavaTimestamp(timestamp); long flowSequence = buf.getUnsignedInt(16); // 16-19 short engineType = buf.getUnsignedByte(20); // 20 short engineId = buf.getUnsignedByte(21); // 21 // the first 2 bits are the sampling mode, the remaining 14 the interval int sampling = buf.getUnsignedShort(22); // 22-23 int samplingInterval = sampling & 0x3FFF; int samplingMode = sampling >> 14; Map<String, Field> headers = new HashMap<>(); headers.put(VERSION, Field.create(version)); headers.put(PACKETID, Field.create(packetId.toString())); headers.put(SENDER, Field.create((sender == null) ? "unknown" : sender.getAddress().toString())); headers.put(LENGTH, Field.create(packetLength)); headers.put(UPTIME, Field.create(uptime)); headers.put(TIMESTAMP, Field.create(timestamp)); headers.put(FLOWSEQ, Field.create(flowSequence)); headers.put(ENGINEID, Field.create(engineId)); headers.put(ENGINETYPE, Field.create(engineType)); headers.put(SAMPLINGINT, Field.create(samplingInterval)); headers.put(SAMPLINGMODE, Field.create(samplingMode)); headers.put(READERID, Field.create(readerId)); for (int i = 0; i < count; i++) { ByteBuf flowBuf = buf.slice(V5_HEADER_SIZE + (i * V5_FLOW_SIZE), V5_FLOW_SIZE); Map<String, Field> fields = new HashMap<>(); fields.putAll(headers); long pkts = flowBuf.getUnsignedInt(16); long bytes = flowBuf.getUnsignedInt(20); fields.put(ID, Field.create(UUIDs.timeBased().toString())); int srcaddr = (int) flowBuf.getUnsignedInt(0); int dstaddr = (int) flowBuf.getUnsignedInt(4); int nexthop = (int) flowBuf.getUnsignedInt(8); fields.put(SRCADDR, Field.create(srcaddr)); fields.put(DSTADDR, Field.create(dstaddr)); fields.put(NEXTHOP, Field.create(nexthop)); fields.put(SRCADDR_S, Field.create(ipToString(srcaddr))); fields.put(DSTADDR_S, Field.create(ipToString(dstaddr))); fields.put(NEXTHOP_S, Field.create(ipToString(nexthop))); fields.put(SRCPORT, Field.create(flowBuf.getUnsignedShort(32))); fields.put(DSTPORT, Field.create(flowBuf.getUnsignedShort(34))); fields.put(SRCAS, Field.create(flowBuf.getUnsignedShort(40))); fields.put(DSTAS, Field.create(flowBuf.getUnsignedShort(42))); fields.put(PACKETS, Field.create(pkts)); fields.put(DOCTECTS, Field.create(bytes)); fields.put(PROTO, Field.create(flowBuf.getUnsignedByte(38))); fields.put(TOS, Field.create(flowBuf.getUnsignedByte(39))); fields.put(TCPFLAGS, Field.create(flowBuf.getUnsignedByte(37))); long first = flowBuf.getUnsignedInt(24); if (first > 0) { fields.put(FIRST, Field.create(timestamp - uptime - first)); } else { fields.put(FIRST, Field.create(0L)); } long last = flowBuf.getUnsignedInt(28); if (last > 0) { fields.put(LAST, Field.create(timestamp - uptime - last)); } else { fields.put(LAST, Field.create(0L)); } fields.put(SNMPINPUT, Field.create(flowBuf.getUnsignedShort(12))); fields.put(SNMPOUTPUT, Field.create(flowBuf.getUnsignedShort(14))); fields.put(SRCMASK, Field.create(flowBuf.getUnsignedByte(44))); fields.put(DSTMASK, Field.create(flowBuf.getUnsignedByte(45))); Record record = context.createRecord(readerId + "::" + recordId++); record.set(Field.create(fields)); result.add(record); } return result; }