Java tutorial
/** * This file is part of Graylog. * * Graylog is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * Graylog is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with Graylog. If not, see <http://www.gnu.org/licenses/>. */ package org.graylog2.plugin; import com.codahale.metrics.Meter; import com.eaio.uuid.UUID; import com.google.common.base.Function; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.net.InetAddresses; import org.graylog2.plugin.streams.Stream; import org.joda.time.DateTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.Nonnull; import java.net.InetAddress; import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; import java.util.stream.Collectors; import static com.google.common.base.Predicates.equalTo; import static com.google.common.base.Predicates.not; import static org.graylog2.plugin.Tools.ES_DATE_FORMAT_FORMATTER; import static org.graylog2.plugin.Tools.buildElasticSearchTimeFormat; import static org.joda.time.DateTimeZone.UTC; public class Message implements Messages { private static final Logger LOG = LoggerFactory.getLogger(Message.class); public static final String FIELD_ID = "_id"; public static final String FIELD_MESSAGE = "message"; public static final String FIELD_FULL_MESSAGE = "full_message"; public static final String FIELD_SOURCE = "source"; public static final String FIELD_TIMESTAMP = "timestamp"; public static final String FIELD_LEVEL = "level"; public static final String FIELD_STREAMS = "streams"; private static final Pattern VALID_KEY_CHARS = Pattern.compile("^[\\w\\.\\-@]*$"); private static final char KEY_REPLACEMENT_CHAR = '_'; public static final ImmutableSet<String> RESERVED_FIELDS = ImmutableSet.of( // ElasticSearch fields. FIELD_ID, "_ttl", "_source", "_all", "_index", "_type", "_score", // Our reserved fields. FIELD_MESSAGE, FIELD_SOURCE, FIELD_TIMESTAMP, "gl2_source_node", "gl2_source_input", "gl2_source_collector", "gl2_source_collector_input", "gl2_remote_ip", "gl2_remote_port", "gl2_remote_hostname", // TODO Due to be removed in Graylog 3.x "gl2_source_radio", "gl2_source_radio_input"); public static final ImmutableSet<String> RESERVED_SETTABLE_FIELDS = ImmutableSet.of(FIELD_MESSAGE, FIELD_SOURCE, FIELD_TIMESTAMP, "gl2_source_node", "gl2_source_input", "gl2_source_radio", "gl2_source_radio_input", "gl2_source_collector", "gl2_source_collector_input", "gl2_remote_ip", "gl2_remote_port", "gl2_remote_hostname"); private static final ImmutableSet<String> REQUIRED_FIELDS = ImmutableSet.of(FIELD_MESSAGE, FIELD_ID); public static final Function<Message, String> ID_FUNCTION = new MessageIdFunction(); private final Map<String, Object> fields = Maps.newHashMap(); private Set<Stream> streams = Sets.newHashSet(); private String sourceInputId; // Used for drools to filter out messages. private boolean filterOut = false; /** * The offset the message originally had in the journal it was read from. This will be MIN_VALUE if no journal * was involved. */ private long journalOffset = Long.MIN_VALUE; private ArrayList<Recording> recordings; public Message(final String message, final String source, final DateTime timestamp) { fields.put(FIELD_ID, new UUID().toString()); addRequiredField(FIELD_MESSAGE, message); addRequiredField(FIELD_SOURCE, source); addRequiredField(FIELD_TIMESTAMP, timestamp); } public Message(final Map<String, Object> fields) { this((String) fields.get(FIELD_ID), Maps.filterKeys(fields, not(equalTo(FIELD_ID)))); } private Message(String id, Map<String, Object> newFields) { Preconditions.checkArgument(id != null, "message id cannot be null"); fields.put(FIELD_ID, id); addFields(newFields); } public boolean isComplete() { for (final String key : REQUIRED_FIELDS) { final Object field = getField(key); if (field == null || (field instanceof String && ((String) field).isEmpty())) { return false; } } return true; } public String getValidationErrors() { final StringBuilder sb = new StringBuilder(); for (String key : REQUIRED_FIELDS) { final Object field = getField(key); if (field == null) { sb.append(key).append(" is missing, "); } else if (field instanceof String && ((String) field).isEmpty()) { sb.append(key).append(" is empty, "); } } return sb.toString(); } public String getId() { return getFieldAs(String.class, FIELD_ID); } public DateTime getTimestamp() { return getFieldAs(DateTime.class, FIELD_TIMESTAMP).withZone(UTC); } public Map<String, Object> toElasticSearchObject(@Nonnull final Meter invalidTimestampMeter) { final Map<String, Object> obj = Maps.newHashMapWithExpectedSize(REQUIRED_FIELDS.size() + fields.size()); for (Map.Entry<String, Object> entry : fields.entrySet()) { final String key = entry.getKey(); // Elasticsearch does not allow "." characters in keys since version 2.0. // See: https://www.elastic.co/guide/en/elasticsearch/reference/2.0/breaking_20_mapping_changes.html#_field_names_may_not_contain_dots if (key != null && key.contains(".")) { final String newKey = key.replace('.', KEY_REPLACEMENT_CHAR); // If the message already contains the transformed key, we skip the field and emit a warning. // This is still not optimal but better than implementing expensive logic with multiple replacement // character options. Conflicts should be rare... if (!obj.containsKey(newKey)) { obj.put(newKey, entry.getValue()); } else { LOG.warn( "Keys must not contain a \".\" character! Ignoring field \"{}\"=\"{}\" in message [{}] - Unable to replace \".\" with a \"{}\" because of key conflict: \"{}\"=\"{}\"", key, entry.getValue(), getId(), KEY_REPLACEMENT_CHAR, newKey, obj.get(newKey)); LOG.debug("Full message with \".\" in message key: {}", this); } } else { if (key != null && obj.containsKey(key)) { final String newKey = key.replace(KEY_REPLACEMENT_CHAR, '.'); // Deliberate warning duplicates because the key with the "." might be transformed before reaching // the duplicate original key with a "_". Otherwise we would silently overwrite the transformed key. LOG.warn( "Keys must not contain a \".\" character! Ignoring field \"{}\"=\"{}\" in message [{}] - Unable to replace \".\" with a \"{}\" because of key conflict: \"{}\"=\"{}\"", newKey, fields.get(newKey), getId(), KEY_REPLACEMENT_CHAR, key, entry.getValue()); LOG.debug("Full message with \".\" in message key: {}", this); } obj.put(key, entry.getValue()); } } obj.put(FIELD_MESSAGE, getMessage()); obj.put(FIELD_SOURCE, getSource()); final Object timestampValue = getField(FIELD_TIMESTAMP); DateTime dateTime; if (timestampValue instanceof Date) { dateTime = new DateTime(timestampValue); } else if (timestampValue instanceof DateTime) { dateTime = (DateTime) timestampValue; } else if (timestampValue instanceof String) { // if the timestamp value is a string, we try to parse it in the correct format. // we fall back to "now", this avoids losing messages which happen to have the wrong timestamp format try { dateTime = ES_DATE_FORMAT_FORMATTER.parseDateTime((String) timestampValue); } catch (IllegalArgumentException e) { LOG.trace("Invalid format for field timestamp '{}' in message {}, forcing to current time.", timestampValue, getId()); invalidTimestampMeter.mark(); dateTime = Tools.nowUTC(); } } else { // don't allow any other types for timestamp, force to "now" LOG.trace("Invalid type for field timestamp '{}' in message {}, forcing to current time.", timestampValue.getClass().getSimpleName(), getId()); invalidTimestampMeter.mark(); dateTime = Tools.nowUTC(); } if (dateTime != null) { obj.put(FIELD_TIMESTAMP, buildElasticSearchTimeFormat(dateTime.withZone(UTC))); } // Manually converting stream ID to string - caused strange problems without it. if (getStreams().isEmpty()) { obj.put(FIELD_STREAMS, Collections.emptyList()); } else { final List<String> streamIds = Lists.newArrayListWithCapacity(streams.size()); for (Stream stream : streams) { streamIds.add(stream.getId()); } obj.put(FIELD_STREAMS, streamIds); } return obj; } @Override public String toString() { final StringBuilder sb = new StringBuilder(); sb.append("source: ").append(getField(FIELD_SOURCE)).append(" | "); final String message = getField(FIELD_MESSAGE).toString().replaceAll("\\n", "").replaceAll("\\t", ""); sb.append("message: "); if (message.length() > 225) { sb.append(message.substring(0, 225)).append(" (...)"); } else { sb.append(message); } sb.append(" { "); final Map<String, Object> filteredFields = Maps.newHashMap(fields); filteredFields.remove(FIELD_SOURCE); filteredFields.remove(FIELD_MESSAGE); Joiner.on(" | ").withKeyValueSeparator(": ").appendTo(sb, filteredFields); sb.append(" }"); return sb.toString(); } public String getMessage() { return getFieldAs(String.class, FIELD_MESSAGE); } public String getSource() { return getFieldAs(String.class, FIELD_SOURCE); } public void setSource(final String source) { fields.put(FIELD_SOURCE, source); } public void addField(final String key, final Object value) { addField(key, value, false); } private void addRequiredField(final String key, final Object value) { addField(key, value, true); } private void addField(final String key, final Object value, final boolean isRequiredField) { final String trimmedKey = key.trim(); // Don't accept protected keys. (some are allowed though lol) if (RESERVED_FIELDS.contains(trimmedKey) && !RESERVED_SETTABLE_FIELDS.contains(trimmedKey) || !validKey(trimmedKey)) { if (LOG.isDebugEnabled()) { LOG.debug("Ignoring invalid or reserved key {} for message {}", trimmedKey, getId()); } return; } if (FIELD_TIMESTAMP.equals(trimmedKey) && value != null && value instanceof Date) { fields.put(FIELD_TIMESTAMP, new DateTime(value)); } else if (value instanceof String) { final String str = ((String) value).trim(); if (isRequiredField || !str.isEmpty()) { fields.put(trimmedKey, str); } } else if (value != null) { fields.put(trimmedKey, value); } } public static boolean validKey(final String key) { return VALID_KEY_CHARS.matcher(key).matches(); } public void addFields(final Map<String, Object> fields) { if (fields == null) { return; } for (Map.Entry<String, Object> field : fields.entrySet()) { addField(field.getKey(), field.getValue()); } } public void addStringFields(final Map<String, String> fields) { if (fields == null) { return; } for (Map.Entry<String, String> field : fields.entrySet()) { addField(field.getKey(), field.getValue()); } } public void addLongFields(final Map<String, Long> fields) { if (fields == null) { return; } for (Map.Entry<String, Long> field : fields.entrySet()) { addField(field.getKey(), field.getValue()); } } public void addDoubleFields(final Map<String, Double> fields) { if (fields == null) { return; } for (Map.Entry<String, Double> field : fields.entrySet()) { addField(field.getKey(), field.getValue()); } } public void removeField(final String key) { if (!RESERVED_FIELDS.contains(key)) { fields.remove(key); } } public <T> T getFieldAs(final Class<T> T, final String key) throws ClassCastException { return T.cast(getField(key)); } public Object getField(final String key) { return fields.get(key); } public Map<String, Object> getFields() { return ImmutableMap.copyOf(fields); } public Iterable<Map.Entry<String, Object>> getFieldsEntries() { return Iterables.unmodifiableIterable(fields.entrySet()); } public int getFieldCount() { return fields.size(); } public boolean hasField(String field) { return fields.containsKey(field); } public Set<String> getFieldNames() { return Collections.unmodifiableSet(fields.keySet()); } @Deprecated public void setStreams(final List<Stream> streams) { this.streams = Sets.newHashSet(streams); } /** * Get the streams this message is currently routed to. * @return an immutable copy of the current set of assigned streams, empty if no streams have been assigned */ public Set<Stream> getStreams() { return ImmutableSet.copyOf(this.streams); } /** * Assign the given stream to this message. * * @param stream the stream to route this message into */ public void addStream(Stream stream) { streams.add(stream); } /** * Assign all of the streams to this message. * @param newStreams an iterable of Stream objects */ public void addStreams(Iterable<Stream> newStreams) { Iterables.addAll(streams, newStreams); } /** * Remove the stream assignment from this message. * @param stream the stream assignment to remove this message from * @return <tt>true</tt> if this message was assigned to the stream */ public boolean removeStream(Stream stream) { return streams.remove(stream); } public List<String> getStreamIds() { if (!hasField(FIELD_STREAMS)) { return streams.stream().map(Stream::getId).collect(Collectors.toList()); } try { @SuppressWarnings("unchecked") final List<String> streamIds = getFieldAs(List.class, FIELD_STREAMS); return new ArrayList<>(streamIds); } catch (ClassCastException e) { LOG.trace("Couldn't cast {} to List", FIELD_STREAMS, e); return Collections.emptyList(); } } public void setFilterOut(final boolean filterOut) { this.filterOut = filterOut; } public boolean getFilterOut() { return this.filterOut; } public void setSourceInputId(String sourceInputId) { this.sourceInputId = sourceInputId; } public String getSourceInputId() { return sourceInputId; } // drools seems to need the "get" prefix public boolean getIsSourceInetAddress() { return fields.containsKey("gl2_remote_ip"); } public InetAddress getInetAddress() { if (!fields.containsKey("gl2_remote_ip")) { return null; } final String ipAddr = (String) fields.get("gl2_remote_ip"); try { return InetAddresses.forString(ipAddr); } catch (IllegalArgumentException ignored) { return null; } } public void setJournalOffset(long journalOffset) { this.journalOffset = journalOffset; } public long getJournalOffset() { return journalOffset; } // helper methods to optionally record timing information per message, useful for debugging or benchmarking // not thread safe! public void recordTiming(ServerStatus serverStatus, String name, long elapsedNanos) { if (shouldNotRecord(serverStatus)) return; lazyInitRecordings(); recordings.add(Recording.timing(name, elapsedNanos)); } public void recordCounter(ServerStatus serverStatus, String name, int counter) { if (shouldNotRecord(serverStatus)) return; lazyInitRecordings(); recordings.add(Recording.counter(name, counter)); } public String recordingsAsString() { if (hasRecordings()) { return Joiner.on(", ").join(recordings); } return ""; } public boolean hasRecordings() { return recordings != null && recordings.size() > 0; } private void lazyInitRecordings() { if (recordings == null) { recordings = new ArrayList<>(); } } private boolean shouldNotRecord(ServerStatus serverStatus) { return !serverStatus.getDetailedMessageRecordingStrategy().shouldRecord(this); } @Override public Iterator<Message> iterator() { if (getFilterOut()) { return Collections.emptyIterator(); } return Iterators.singletonIterator(this); } public static abstract class Recording { public static Timing timing(String name, long elapsedNanos) { return new Timing(name, elapsedNanos); } public static Counter counter(String name, int counter) { return new Counter(name, counter); } } private static class Timing extends Recording { private final String name; private final long elapsedNanos; public Timing(String name, long elapsedNanos) { this.name = name; this.elapsedNanos = elapsedNanos; } @Override public String toString() { return name + ": " + TimeUnit.NANOSECONDS.toMicros(elapsedNanos) + "micros"; } } private static class Counter extends Recording { private final String name; private final int counter; public Counter(String name, int counter) { this.name = name; this.counter = counter; } @Override public String toString() { return name + ": " + counter; } } public static class MessageIdFunction implements Function<Message, String> { @Override public String apply(final Message input) { return input.getId(); } } }