Java tutorial
/* * Copyright 2012 Nodeable Inc * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.streamreduce.core.service; import com.google.common.base.Objects; import com.google.common.base.Preconditions; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.common.cache.RemovalListener; import com.google.common.cache.RemovalNotification; import com.google.common.collect.ImmutableSet; import com.mongodb.BasicDBObject; import com.mongodb.DBObject; import com.streamreduce.ConnectionNotFoundException; import com.streamreduce.Constants; import com.streamreduce.ProviderIdConstants; import com.streamreduce.connections.ConnectionProviderFactory; import com.streamreduce.core.CommandNotAllowedException; import com.streamreduce.core.dao.DAODatasourceType; import com.streamreduce.core.dao.GenericCollectionDAO; import com.streamreduce.core.dao.InventoryItemDAO; import com.streamreduce.core.event.EventId; import com.streamreduce.core.model.Connection; import com.streamreduce.core.model.Event; import com.streamreduce.core.model.InventoryItem; import com.streamreduce.core.model.SobaObject; import com.streamreduce.core.model.User; import com.streamreduce.core.model.messages.MessageType; import com.streamreduce.core.model.messages.details.feed.FeedEntryDetails; import com.streamreduce.core.model.messages.details.jira.JiraActivityDetails; import com.streamreduce.core.model.messages.details.pingdom.PingdomEntryDetails; import com.streamreduce.core.model.messages.details.twitter.TwitterActivityDetails; import com.streamreduce.core.service.exception.InvalidCredentialsException; import com.streamreduce.core.service.exception.InventoryItemNotFoundException; import com.streamreduce.util.AWSClient; import com.streamreduce.util.ExternalIntegrationClient; import com.streamreduce.util.FeedClient; import com.streamreduce.util.GitHubClient; import com.streamreduce.util.GoogleAnalyticsClient; import com.streamreduce.util.HashtagUtil; import com.streamreduce.util.JSONUtils; import com.streamreduce.util.JiraClient; import com.streamreduce.util.MessageUtils; import com.streamreduce.util.PingdomClient; import com.streamreduce.util.TwitterClient; import com.sun.syndication.feed.synd.SyndContent; import com.sun.syndication.feed.synd.SyndEntry; import com.sun.syndication.feed.synd.SyndFeed; import com.sun.syndication.io.SyndFeedInput; import com.sun.syndication.io.XmlReader; import net.sf.json.JSONArray; import net.sf.json.JSONObject; import org.apache.abdera.model.Element; import org.apache.abdera.model.Entry; import org.bson.types.ObjectId; import org.codehaus.jackson.map.ObjectMapper; import org.jclouds.aws.ec2.domain.Tag; import org.jclouds.cloudwatch.CloudWatchAsyncApi; import org.jclouds.cloudwatch.CloudWatchApi; import org.jclouds.cloudwatch.domain.*; import org.jclouds.cloudwatch.features.MetricApi; import org.jclouds.compute.domain.ComputeType; import org.jclouds.compute.domain.NodeMetadata; import org.jclouds.domain.Location; import org.jclouds.domain.LocationBuilder; import org.jclouds.domain.LocationScope; import org.jclouds.rest.RestContext; import org.joda.time.DateTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import java.io.IOException; import java.net.URI; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; import java.util.Collections; import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; import javax.xml.namespace.QName; /** * Implementation of {@link InventoryService}. */ @Service("inventoryService") public class InventoryServiceImpl implements InventoryService { protected transient Logger logger = LoggerFactory.getLogger(getClass()); private Map<String, Unit> ec2CloudWatchMetricNames = null; private Set<Statistics> ec2CloudWatchStatisticsSet = ImmutableSet.of(Statistics.AVERAGE, Statistics.MINIMUM, Statistics.MAXIMUM); private final Cache<ObjectId, ExternalIntegrationClient> externalClientCache = CacheBuilder.newBuilder() .expireAfterWrite(3, TimeUnit.MINUTES) .removalListener(new RemovalListener<ObjectId, ExternalIntegrationClient>() { /** * {@inheritDoc} */ @Override public void onRemoval(RemovalNotification<ObjectId, ExternalIntegrationClient> n) { ExternalIntegrationClient client = n.getValue(); if (client != null) { logger.debug("Cleaning up ExternalIntegrationClient [" + client.getConnectionId() + "]"); client.cleanUp(); } } }).build(); @Autowired InventoryItemDAO inventoryItemDAO; @Autowired GenericCollectionDAO genericCollectionDAO; @Autowired EmailService emailService; @Autowired EventService eventService; @Autowired MessageService messageService; @Autowired ConnectionService connectionService; @Autowired ConnectionProviderFactory connectionProviderFactory; /** * {@inheritDoc} */ @Override public BasicDBObject getInventoryItemPayload(InventoryItem inventoryItem) { return genericCollectionDAO.getById(DAODatasourceType.BUSINESS, Constants.INVENTORY_ITEM_METADATA_COLLECTION_NAME, inventoryItem.getMetadataId()); } /** * {@inheritDoc} */ @Override public InventoryItem createInventoryItem(Connection connection, JSONObject json) throws ConnectionNotFoundException, InvalidCredentialsException, IOException { Preconditions.checkNotNull(connection, "connection cannot be null."); Preconditions.checkNotNull(json, "json cannot be null."); String providerId = connection.getProviderId(); InventoryItem inventoryItem = new InventoryItem(); inventoryItem.setAccount(connection.getAccount()); inventoryItem.setUser(connection.getUser()); inventoryItem.setConnection(connection); inventoryItem.addHashtag(connection.getProviderId()); inventoryItem.setDeleted(false); // Default to visibility of the connection (Should be overridden below if required) inventoryItem.setVisibility(connection.getVisibility()); // No other way... polymorphism is Hard... if (providerId.equals(ProviderIdConstants.AWS_PROVIDER_ID)) { extendAWSInventoryItem(inventoryItem, json); } else if (providerId.equals(ProviderIdConstants.GITHUB_PROVIDER_ID)) { extendGitHubInventoryItem(inventoryItem, json); } else if (providerId.equals(ProviderIdConstants.GOOGLE_ANALYTICS_PROVIDER_ID)) { extendGoogleAnalyticsInventoryItem(inventoryItem, json); } else if (providerId.equals(ProviderIdConstants.JIRA_PROVIDER_ID)) { extendJiraInventoryItem(inventoryItem, json); } else if (providerId.equals(ProviderIdConstants.PINGDOM_PROVIDER_ID)) { extendPingdomInventoryItem(inventoryItem, json); } else if (providerId.equals(ProviderIdConstants.CUSTOM_PROVIDER_ID)) { extendGenericInventoryItem(inventoryItem, json); } else if (providerId.equals(ProviderIdConstants.NAGIOS_PROVIDER_ID)) { extendGenericInventoryItem(inventoryItem, json); } else { throw new IllegalArgumentException(providerId + " does not support creating inventory items."); } // Create metadata DBObject metadataEntry = genericCollectionDAO.createCollectionEntry(DAODatasourceType.BUSINESS, Constants.INVENTORY_ITEM_METADATA_COLLECTION_NAME, json.toString()); inventoryItem.setMetadataId((ObjectId) metadataEntry.get("_id")); // Persist the inventory item inventoryItemDAO.save(inventoryItem); // Create the event Event event = eventService.createEvent(EventId.CREATE, inventoryItem, null); // Create the message messageService.sendInventoryMessage(event, inventoryItem); return inventoryItem; } /** * {@inheritDoc} */ @Override public InventoryItem updateInventoryItem(InventoryItem inventoryItem, JSONObject json) throws ConnectionNotFoundException, InvalidCredentialsException, IOException { Preconditions.checkNotNull(inventoryItem, "inventoryItem cannot be null."); Preconditions.checkNotNull(json, "json cannot be null."); Connection connection = inventoryItem.getConnection(); String providerId = connection.getProviderId(); // No other way... if (providerId.equals(ProviderIdConstants.AWS_PROVIDER_ID)) { extendAWSInventoryItem(inventoryItem, json); } else if (providerId.equals(ProviderIdConstants.GITHUB_PROVIDER_ID)) { extendGitHubInventoryItem(inventoryItem, json); } else if (providerId.equals(ProviderIdConstants.GOOGLE_ANALYTICS_PROVIDER_ID)) { extendGoogleAnalyticsInventoryItem(inventoryItem, json); } else if (providerId.equals(ProviderIdConstants.JIRA_PROVIDER_ID)) { extendJiraInventoryItem(inventoryItem, json); } else if (providerId.equals(ProviderIdConstants.CUSTOM_PROVIDER_ID)) { extendGenericInventoryItem(inventoryItem, json); } else if (providerId.equals(ProviderIdConstants.NAGIOS_PROVIDER_ID)) { extendGenericInventoryItem(inventoryItem, json); } else if (providerId.equals(ProviderIdConstants.PINGDOM_PROVIDER_ID)) { extendPingdomInventoryItem(inventoryItem, json); } else { throw new IllegalArgumentException(providerId + " does not support creating inventory items."); } // Change visibility if necessary? // Update metadata try { genericCollectionDAO.updateCollectionEntry(DAODatasourceType.BUSINESS, Constants.INVENTORY_ITEM_METADATA_COLLECTION_NAME, inventoryItem.getMetadataId(), json.toString()); } catch (Exception e) { // Should never happen logger.error("Error updating project hosting inventory item cache: " + inventoryItem.getId(), e); } // Send a silent update if none of our internal properties have changed boolean silentUpdate = false; InventoryItem oldInventoryItem; try { oldInventoryItem = getInventoryItem(inventoryItem.getId()); } catch (InventoryItemNotFoundException e) { // Should never happen but just in case oldInventoryItem = null; } // Be silent unless alias, description, hashtags or visibility are different. (None of the other properties // should be changeable externally or via our exposed APIs.) if (oldInventoryItem != null) { silentUpdate = (Objects.equal(oldInventoryItem.getAlias(), inventoryItem.getAlias()) && Objects.equal(oldInventoryItem.getDescription(), inventoryItem.getDescription()) && Objects.equal(oldInventoryItem.getHashtags(), inventoryItem.getHashtags()) && Objects.equal(oldInventoryItem.getVisibility(), inventoryItem.getVisibility())); } // Persist the inventory item return updateInventoryItem(inventoryItem, silentUpdate); } /** * {@inheritDoc} */ @Override public InventoryItem updateInventoryItem(InventoryItem inventoryItem, boolean silentUpdate) { Preconditions.checkNotNull(inventoryItem, "inventoryItem cannot be null."); inventoryItemDAO.save(inventoryItem); if (!silentUpdate) { // Create the event Event event = eventService.createEvent(EventId.UPDATE, inventoryItem, null); // Create the message messageService.sendInventoryMessage(event, inventoryItem); } return inventoryItem; } /** * {@inheritDoc} */ @Override public void deleteInventoryItem(InventoryItem inventoryItem) { Preconditions.checkNotNull(inventoryItem, "inventoryItem cannot be null."); // Delete the metadata entry genericCollectionDAO.removeCollectionEntry(DAODatasourceType.BUSINESS, Constants.INVENTORY_ITEM_METADATA_COLLECTION_NAME, inventoryItem.getMetadataId()); inventoryItemDAO.delete(inventoryItem); // If the item was already marked as deleted, do not resend the event/message if (inventoryItem.isDeleted()) { // Create the event Event event = eventService.createEvent(EventId.DELETE, inventoryItem, null); // Create the message messageService.sendInventoryMessage(event, inventoryItem); } } /** * {@inheritDoc} */ @Override public void markInventoryItemDeleted(InventoryItem inventoryItem) { inventoryItem.setDeleted(true); inventoryItemDAO.save(inventoryItem); // Create the event Event event = eventService.createEvent(EventId.DELETE, inventoryItem, null); // Create the message messageService.sendInventoryMessage(event, inventoryItem); } /** * {@inheritDoc} */ @Override public List<InventoryItem> getInventoryItems(Connection connection) { return getInventoryItems(connection, null); } /** * {@inheritDoc} */ @Override public List<InventoryItem> getInventoryItems(Connection connection, User user) { List<InventoryItem> inventoryItems = inventoryItemDAO.getInventoryItems(connection, user); for (InventoryItem inventoryItem : inventoryItems) { // Create the event eventService.createEvent(EventId.READ, inventoryItem, null); } return inventoryItems; } /** * {@inheritDoc} */ @Override public List<InventoryItem> getInventoryItems(ObjectId connectionId) { List<InventoryItem> inventoryItems = inventoryItemDAO.getInventoryItems(connectionId); for (InventoryItem inventoryItem : inventoryItems) { // Create the event eventService.createEvent(EventId.READ, inventoryItem, null); } return inventoryItems; } /** * {@inheritDoc} */ @Override public List<InventoryItem> getInventoryItemsForExternalId(String externalId) { List<InventoryItem> inventoryItems = inventoryItemDAO.getByExternalIdNotDeleted(externalId); for (InventoryItem inventoryItem : inventoryItems) { // Create the event eventService.createEvent(EventId.READ, inventoryItem, null); } return inventoryItems; } /** * {@inheritDoc} */ @Override public InventoryItem getInventoryItem(ObjectId objectId) throws InventoryItemNotFoundException { InventoryItem inventoryItem = inventoryItemDAO.get(objectId); if (inventoryItem == null) { throw new InventoryItemNotFoundException(objectId.toString()); } // Create the event eventService.createEvent(EventId.READ, inventoryItem, null); return inventoryItem; } /** * {@inheritDoc} */ @Override public InventoryItem getInventoryItemForExternalId(Connection connection, String externalId) throws InventoryItemNotFoundException { Preconditions.checkNotNull(connection, "connection cannot be null."); Preconditions.checkNotNull(externalId, "externalId cannot be null."); InventoryItem inventoryItem = inventoryItemDAO.getInventoryItem(connection, externalId); if (inventoryItem == null) { throw new InventoryItemNotFoundException("No inventory item for connection [" + connection.getId() + "] " + "with the given externalId of " + externalId); } // Create the event eventService.createEvent(EventId.READ, inventoryItem, null); return inventoryItem; } /** * {@inheritDoc} */ @Override public void refreshInventoryItemCache(Connection connection) throws ConnectionNotFoundException, InvalidCredentialsException, IOException { Preconditions.checkNotNull(connection, "connection cannot be null."); logger.debug("Updating inventory item cache for connection [" + connection.getId() + "]: " + connection.getAlias()); List<String> processedKeys = new ArrayList<>(); List<JSONObject> externalInventoryItems; ExternalIntegrationClient client = getClient(connection); if (client instanceof AWSClient) { AWSClient awsClient = null; try { awsClient = new AWSClient(connection); // Get the EC2 inventory items externalInventoryItems = (awsClient.getEC2Instances()); // Get the S3 inventory items externalInventoryItems.addAll(awsClient.getS3BucketsAsJson()); } finally { if (awsClient != null) { awsClient.cleanUp(); } } } else if (client instanceof GitHubClient) { externalInventoryItems = ((GitHubClient) client).getRepositories(); } else if (client instanceof GoogleAnalyticsClient) { externalInventoryItems = ((GoogleAnalyticsClient) client).getProfiles(); } else if (client instanceof JiraClient) { externalInventoryItems = ((JiraClient) client).getProjects(false); } else if (client instanceof PingdomClient) { externalInventoryItems = ((PingdomClient) client).checks(); } else if (client instanceof FeedClient) { return; } else if (client instanceof TwitterClient) { return; } else { throw new IllegalArgumentException( client.getClass().getName() + " is not a supported external client."); } logger.debug(" Provider id: " + connection.getProviderId()); logger.debug(" Inventory items found: " + externalInventoryItems.size()); for (JSONObject json : externalInventoryItems) { InventoryItem inventoryItem = null; JSONObject externalInventoryItemAsJSON; String externalId; if (client instanceof AWSClient) { String type = json.getString("type"); if (type.equals(ComputeType.NODE.toString())) { externalId = json.getString("providerId"); } else { externalId = json.getString("name"); } externalInventoryItemAsJSON = json; } else if (client instanceof GitHubClient) { externalId = json.getJSONObject("owner").getString("login") + "/" + json.getString("name"); externalInventoryItemAsJSON = json; } else if (client instanceof PingdomClient) { externalId = json.getString("id"); externalInventoryItemAsJSON = json; } else if (client instanceof GoogleAnalyticsClient) { externalId = json.getString("id"); externalInventoryItemAsJSON = json; } else { externalId = json.getString("key"); try { externalInventoryItemAsJSON = ((JiraClient) client).getProjectDetails(externalId); } catch (InvalidCredentialsException | IOException e) { // This should never happen as we've already validated the connection by this point logger.warn("Unable to get the Jira project details for " + externalId + ": " + e.getMessage()); return; } } try { inventoryItem = getInventoryItemForExternalId(connection, externalId); } catch (InventoryItemNotFoundException e) { // This is more than possible and is handled below } if (inventoryItem == null || inventoryItem.isDeleted()) { if (inventoryItem != null && inventoryItem.isDeleted()) { // Permanently remove the current inventory item marked as deleted. This can happen if you mark // an inventory item as deleted (due to external deletion) and then recreate the same inventory // item externally before the clean process (ran after refresh) could occur. deleteInventoryItem(inventoryItem); } createInventoryItem(connection, externalInventoryItemAsJSON); } else if (!processedKeys.contains(externalId)) { updateInventoryItem(inventoryItem, externalInventoryItemAsJSON); } processedKeys.add(externalId); } // Handle inventory items deleted externally List<InventoryItem> inventoryItems = getInventoryItems(connection); for (InventoryItem inventoryItem : inventoryItems) { if (processedKeys.contains(inventoryItem.getExternalId())) { continue; } // Just mark the item as deleted. Future polling jobs will remove the inventory item marked as deleted // when appropriate. if (!inventoryItem.isDeleted()) { markInventoryItemDeleted(inventoryItem); } } } /** * {@inheritDoc} */ @Override public void pullInventoryItemActivity(Connection connection) throws ConnectionNotFoundException, InvalidCredentialsException, IOException { Preconditions.checkNotNull(connection, "connection cannot be null."); String providerId = connection.getProviderId(); // TODO: pullGitHubActivity and pullJiraActivity could be simplified // No other way... if (providerId.equals(ProviderIdConstants.AWS_PROVIDER_ID)) { pullEC2CloudWatchMetrics(connection); // AWS S3 doesn't do CloudWatch metrics so we can tackle "metrics" when we do the same for GitHub/Jira } else if (providerId.equals(ProviderIdConstants.GITHUB_PROVIDER_ID)) { pullGitHubActivity(connection); } else if (providerId.equals(ProviderIdConstants.GOOGLE_ANALYTICS_PROVIDER_ID)) { pullGoogleAnalyticsActivity(connection); } else if (providerId.equals(ProviderIdConstants.JIRA_PROVIDER_ID)) { pullJiraActivity(connection); } else if (providerId.equals(ProviderIdConstants.PINGDOM_PROVIDER_ID)) { pullPingdomActivity(connection); } else if (providerId.equals(ProviderIdConstants.FEED_PROVIDER_ID)) { pullFeedActivity(connection); } else if (providerId.equals(ProviderIdConstants.TWITTER_PROVIDER_ID)) { pullTwitterActivity(connection); } else { throw new IllegalArgumentException(providerId + " does not support polling for activity."); } } /* HELPERS METHODS */ /** * {@inheritDoc} */ @Override public String getComputeInstanceIPAddress(InventoryItem inventoryItem) { Preconditions.checkNotNull(inventoryItem, "inventoryItem cannot be null."); Preconditions.checkArgument(inventoryItem.getType().equals(Constants.COMPUTE_INSTANCE_TYPE), "Only inventory items of type '" + Constants.COMPUTE_INSTANCE_TYPE + "' can have an OS name."); DBObject nodeMetadata = genericCollectionDAO.getById(DAODatasourceType.BUSINESS, Constants.INVENTORY_ITEM_METADATA_COLLECTION_NAME, inventoryItem.getMetadataId()); return getComputeInstanceIPAddress( nodeMetadata != null ? JSONObject.fromObject(nodeMetadata.toString()) : null); } /** * {@inheritDoc} */ @Override public String getComputeInstanceIPAddress(JSONObject json) { Preconditions.checkNotNull(json, "json cannot be null."); Preconditions.checkArgument(json.getString("type").equals(ComputeType.NODE.toString()), "Only inventory items of type '" + Constants.COMPUTE_INSTANCE_TYPE + "' can have an OS name."); String ipAddress = null; if (json.containsKey("publicAddresses")) { JSONArray publicAddresses = json.getJSONArray("publicAddresses"); // TODO: How do we want to handle multiple IP addresses? if (publicAddresses.size() > 0) { ipAddress = (String) publicAddresses.get(0); } } return ipAddress; } /** * {@inheritDoc} */ @Override public String getComputeInstanceOSName(InventoryItem inventoryItem) { Preconditions.checkNotNull(inventoryItem, "inventoryItem cannot be null."); Preconditions.checkArgument(inventoryItem.getType().equals(Constants.COMPUTE_INSTANCE_TYPE), "Only inventory items of type '" + Constants.COMPUTE_INSTANCE_TYPE + "' can have an OS name."); DBObject nodeMetadata = genericCollectionDAO.getById(DAODatasourceType.BUSINESS, Constants.INVENTORY_ITEM_METADATA_COLLECTION_NAME, inventoryItem.getMetadataId()); return getComputeInstanceOSName( nodeMetadata != null ? JSONObject.fromObject(nodeMetadata.toString()) : null); } /** * {@inheritDoc} */ @Override public String getComputeInstanceOSName(JSONObject json) { Preconditions.checkNotNull(json, "json cannot be null."); Preconditions.checkArgument(json.getString("type").equals(ComputeType.NODE.toString()), "Only inventory items of type '" + Constants.COMPUTE_INSTANCE_TYPE + "' can have an OS name."); String osName = "UNKNOWN"; if (json.containsKey("operatingSystem")) { JSONObject operatingSystem = json.getJSONObject("operatingSystem"); if (operatingSystem != null && operatingSystem.containsKey("family")) { String rawOSName = operatingSystem.getString("family"); if (rawOSName != null && !(rawOSName.equalsIgnoreCase("UNRECOGNIZED"))) { osName = rawOSName; } } } return osName; } /** * {@inheritDoc} */ @Override public Location getLocationByScope(InventoryItem inventoryItem, LocationScope scope) { Preconditions.checkNotNull(inventoryItem, "inventoryItem cannot be null."); Preconditions.checkNotNull(scope, "scope cannot be null."); DBObject nodeMetadata = genericCollectionDAO.getById(DAODatasourceType.BUSINESS, Constants.INVENTORY_ITEM_METADATA_COLLECTION_NAME, inventoryItem.getMetadataId()); return getLocationByScope(nodeMetadata != null ? JSONObject.fromObject(nodeMetadata.toString()) : null, scope); } /** * {@inheritDoc} */ @Override public Location getLocationByScope(JSONObject json, LocationScope scope) { Preconditions.checkNotNull(json, "json cannot be null."); Preconditions.checkNotNull(scope, "scope cannot be null."); Preconditions.checkArgument(json.containsKey("location"), "json must contain a 'location' attribute."); JSONObject locationObject = json.containsKey("location") && json.get("location") != null ? json.getJSONObject("location") : null; Location location = null; while (locationObject != null) { if (locationObject.containsKey("scope")) { String locationScope = locationObject.getString("scope"); if (locationScope != null) { if (scope == LocationScope.valueOf(locationScope)) { location = buildLocationFromJSON(locationObject); break; } } } locationObject = locationObject.containsKey("parent") && locationObject.get("parent") != null ? locationObject.getJSONObject("parent") : null; } return location; } /** * {@inheritDoc} */ @Override public void addHashtag(InventoryItem inventoryItem, SobaObject tagger, String tag) { inventoryItem.addHashtag(tag); handleHashtagEvent(EventId.HASHTAG_ADD, inventoryItem, tagger, tag); updateInventoryItem(inventoryItem, true); } /** * {@inheritDoc} */ @Override public void removeHashtag(InventoryItem inventoryItem, SobaObject tagger, String tag) { inventoryItem.removeHashtag(tag); handleHashtagEvent(EventId.HASHTAG_DELETE, inventoryItem, tagger, tag); updateInventoryItem(inventoryItem, true); } /** * {@inheritDoc} */ @Override public void rebootComputeInstance(InventoryItem inventoryItem) throws CommandNotAllowedException, InvalidCredentialsException { Preconditions.checkNotNull(inventoryItem, "inventoryItem cannot be null."); Preconditions.checkArgument(inventoryItem.getType().equals(Constants.COMPUTE_INSTANCE_TYPE), "Inventory item of type '" + inventoryItem.getType() + "' cannot be rebooted."); AWSClient client = (AWSClient) getClient(inventoryItem.getConnection()); logger.debug("Rebooting node: " + inventoryItem.getExternalId()); BasicDBObject payload = getInventoryItemPayload(inventoryItem); String jcloudsNodeId = payload.getString("id"); NodeMetadata nodeMetadata = client.getEC2Instance(jcloudsNodeId); if (nodeMetadata.getStatus().equals(NodeMetadata.Status.TERMINATED)) { throw new CommandNotAllowedException("You cannot reboot a terminated node."); } EventId eventId; if (client.rebootEC2Instance(jcloudsNodeId)) { eventId = EventId.CLOUD_INVENTORY_ITEM_REBOOT; } else { // TODO: Handle this issue but it can be a false positive if the time it takes surpasses the time we wait eventId = EventId.CLOUD_INVENTORY_ITEM_REBOOT_FAILURE; } // Create the event Event event = eventService.createEvent(eventId, inventoryItem, null); // Create the message messageService.sendInventoryMessage(event, inventoryItem); } /** * {@inheritDoc} */ @Override public void destroyComputeInstance(InventoryItem inventoryItem) throws InvalidCredentialsException { Preconditions.checkNotNull(inventoryItem, "inventoryItem cannot be null."); Preconditions.checkArgument(inventoryItem.getType().equals(Constants.COMPUTE_INSTANCE_TYPE), "Inventory item of type '" + inventoryItem.getType() + "' cannot be destroyed."); AWSClient client = (AWSClient) getClient(inventoryItem.getConnection()); logger.debug("Terminating node: " + inventoryItem.getExternalId()); BasicDBObject payload = getInventoryItemPayload(inventoryItem); String jcloudsNodeId = payload.getString("id"); NodeMetadata nodeMetadata = client.getEC2Instance(jcloudsNodeId); if (nodeMetadata.getStatus().equals(NodeMetadata.Status.TERMINATED)) { return; } EventId eventId; if (client.destroyEC2Instance(jcloudsNodeId)) { eventId = EventId.CLOUD_INVENTORY_ITEM_TERMINATE; } else { // TODO: Handle this issue but it can be a false positive if the time it takes surpasses the time we wait eventId = EventId.CLOUD_INVENTORY_ITEM_TERMINATE_FAILURE; } // Create the event Event event = eventService.createEvent(eventId, inventoryItem, null); // Create the message messageService.sendInventoryMessage(event, inventoryItem); } /* PRIVATE METHODS */ /** * Takes the passed in JSON and creates a {@link Location} from it. (Does not include parents or children.) * * @param json the JSON that should represent a location * * @return the location */ private Location buildLocationFromJSON(JSONObject json) { LocationBuilder builder = new LocationBuilder(); Set<String> iso3166Codes = new HashSet<>(); Map<String, Object> metadata = new HashMap<>(); if (json.containsKey("iso3166Codes")) { for (Object rawCode : json.getJSONArray("iso3166Codes")) { iso3166Codes.add((String) rawCode); } } if (json.containsKey("metadata")) { JSONObject rawMetadata = json.getJSONObject("metadata"); for (Object rawKey : rawMetadata.keySet()) { metadata.put((String) rawKey, rawMetadata.get(rawKey)); } } builder.description(json.containsKey("description") ? json.getString("description") : null) .iso3166Codes(iso3166Codes).id(json.containsKey("id") ? json.getString("id") : null) .scope(json.containsKey("scope") ? LocationScope.valueOf(json.getString("scope")) : null) .metadata(metadata); return builder.build(); } /** * Takes the inventory item already constructed in * {@link #createInventoryItem(com.streamreduce.core.model.Connection, net.sf.json.JSONObject)} and updates its * information with data in JSON. * * @param inventoryItem the inventory item * @param json the json representing the inventory item (from jclouds) * * @throws InvalidCredentialsException if the connection's credentials are invalid */ private void extendAWSInventoryItem(InventoryItem inventoryItem, JSONObject json) throws InvalidCredentialsException { String externalType = json.getString("type"); Connection connection = inventoryItem.getConnection(); String externalId = inventoryItem.getExternalId(); String name = inventoryItem.getAlias(); String internalType; if (externalId == null) { if (externalType.equals(ComputeType.NODE.toString())) { externalId = json.getString("providerId"); } else { externalId = json.getString("name"); } } if (externalType.equals(ComputeType.NODE.toString())) { name = json.getString("name"); AWSClient client = null; try { client = new AWSClient(inventoryItem.getConnection()); Set<Tag> ec2Tags = client.getEC2InstanceTags(externalId); // Handle adding new hashtags based on the EC2 tags and the instance name for (Tag tag : ec2Tags) { String key = tag.getKey(); String value = tag.getValue(); if (key.equals("Name")) { if (value.length() > 0) { name = value; } } else { inventoryItem.addHashtag(key + "=" + value); } } } finally { if (client != null) { client.cleanUp(); } } internalType = Constants.COMPUTE_INSTANCE_TYPE; } else { internalType = Constants.BUCKET_TYPE; } if (!StringUtils.hasText(name)) { name = externalId; } inventoryItem.setAlias(name); inventoryItem.setExternalId(externalId); inventoryItem.setType(internalType); inventoryItem.addHashtag(externalId); inventoryItem.addHashtag(internalType); // For all of AWS, we add hashtags for the availability zone and region Location region = getLocationByScope(json, LocationScope.REGION); Location zone = getLocationByScope(json, LocationScope.ZONE); if (region != null) { inventoryItem.addHashtag(region.getId()); } if (zone != null) { inventoryItem.addHashtag(zone.getId()); } // For AWS EC2, we add OS name if (externalType.equals(ComputeType.NODE.toString())) { String osName = getComputeInstanceOSName(json); if (osName != null) { inventoryItem.addHashtag(getComputeInstanceOSName(json)); } } } /** * Takes the inventory item already constructed in * {@link #createInventoryItem(com.streamreduce.core.model.Connection, net.sf.json.JSONObject)} and updates its * information with data in JSON. * * @param inventoryItem the inventory item * @param json the json representing the inventory item (from jclouds) */ private void extendGitHubInventoryItem(InventoryItem inventoryItem, JSONObject json) { boolean isFork = json.getBoolean("fork"); boolean isPrivate = json.getBoolean("private"); String language = json.getString("language"); String name = json.getString("name"); String owner = json.getJSONObject("owner").getString("login"); String externalId = owner + "/" + name; String description = json.getString("description"); if (description.length() > 256) { description = description.substring(0, 255); } inventoryItem.setAlias(externalId); inventoryItem.setDescription(description); inventoryItem.setExternalId(externalId); inventoryItem.setType(Constants.PROJECT_TYPE); // Override default visibility, we default to hidden if the repository is private if (isPrivate) { inventoryItem.setVisibility(SobaObject.Visibility.SELF); } // For GitHub, we add hashtags indicating the language and whether or not it is a fork if (isFork) { inventoryItem.addHashtag("fork"); } if (StringUtils.hasText(language) && !language.equals("null")) { inventoryItem.addHashtag(language); } inventoryItem.addHashtag(inventoryItem.getType()); } /** * Takes the inventory item already constructed in * {@link #createInventoryItem(com.streamreduce.core.model.Connection, net.sf.json.JSONObject)} and updates its * information with data in JSON. * * @param inventoryItem the inventory item * @param json the json representing the inventory item (from Google Analytics) */ private void extendGoogleAnalyticsInventoryItem(InventoryItem inventoryItem, JSONObject json) { String name = json.getString("name"); String externalId = json.getString("id"); inventoryItem.setAlias(name); inventoryItem.setExternalId(externalId); inventoryItem.setType(Constants.ANALYTICS_TYPE); inventoryItem.addHashtag(inventoryItem.getType()); } /** * Takes the inventory item already constructed in * {@link #createInventoryItem(com.streamreduce.core.model.Connection, net.sf.json.JSONObject)} and updates its * information with data in JSON. * * @param inventoryItem the inventory item * @param json the json representing the inventory item (from jclouds) * * @throws InvalidCredentialsException if the connection's credentials are invalid */ private void extendJiraInventoryItem(InventoryItem inventoryItem, JSONObject json) throws InvalidCredentialsException, IOException { Connection connection = inventoryItem.getConnection(); ObjectId connectionId = connection.getId(); String key = json.getString("key"); String name = json.getString("name"); String description = json.getString("description"); ExternalIntegrationClient rawClient = externalClientCache.getIfPresent(connectionId); JiraClient client; if (rawClient == null) { client = new JiraClient(connection); externalClientCache.put(connectionId, client); } else { client = (JiraClient) rawClient; } if (description.length() > 256) { description = description.substring(0, 255); } inventoryItem.setAlias(name != null ? name : key); inventoryItem.setDescription(description); inventoryItem.setExternalId(key); inventoryItem.setType(Constants.PROJECT_TYPE); // Override default visibility, we default to hidden if the project is not public if (!client.isProjectPublic(key)) { inventoryItem.setVisibility(SobaObject.Visibility.SELF); } inventoryItem.addHashtag(inventoryItem.getType()); // We are unable to gather any extra hashtags to apply to Jira projects at this time } /** * Takes the inventory item already constructed in * {@link #createInventoryItem(com.streamreduce.core.model.Connection, net.sf.json.JSONObject)} and updates its * information with data in JSON. * * @param inventoryItem the inventory item * @param json the json representing the inventory item (from jclouds) * * @throws InvalidCredentialsException if the connection's credentials are invalid */ private void extendPingdomInventoryItem(InventoryItem inventoryItem, JSONObject json) { String externalId = json.getString("id"); inventoryItem.setAlias(json.getString("name")); inventoryItem.setDescription(json.getString("hostname")); inventoryItem.setExternalId(externalId); inventoryItem.setType(Constants.MONITOR_TYPE); inventoryItem.addHashtag(json.getString("type")); // http, dns, tcp, etc... inventoryItem.addHashtag(inventoryItem.getType()); } /** * Takes the inventory item already constructed in * {@link #createInventoryItem(com.streamreduce.core.model.Connection, net.sf.json.JSONObject)} and updates its * information with data in JSON. * * @param inventoryItem the inventory item * @param json the json representing the inventory item (from jclouds) * * @throws InvalidCredentialsException if the connection's credentials are invalid */ private void extendGenericInventoryItem(InventoryItem inventoryItem, JSONObject json) { String externalId = json.getString("inventoryItemId"); JSONArray hashtags = json.containsKey("hashtags") ? json.getJSONArray("hashtags") : new JSONArray(); if (!StringUtils.hasText(inventoryItem.getAlias())) { inventoryItem.setAlias(externalId); } if (!StringUtils.hasText(inventoryItem.getDescription())) { inventoryItem.setDescription("Created automatically from IMG activity message."); } inventoryItem.setExternalId(externalId); inventoryItem.setType(Constants.CUSTOM_TYPE); // Should we support the user specifying a type instead of hard coding? for (Object rawHashtag : hashtags) { if (rawHashtag != null && !rawHashtag.toString().equals("null")) { inventoryItem.addHashtag(HashtagUtil.normalizeTag(rawHashtag.toString())); } } inventoryItem.addHashtag(inventoryItem.getType()); } /** * Simple helper to send hashtag events whenever * {@link #addHashtag(com.streamreduce.core.model.Taggable, com.streamreduce.core.model.SobaObject, String)} or * {@link #removeHashtag(com.streamreduce.core.model.Taggable, com.streamreduce.core.model.SobaObject, String)} is called. * * @param eventId the event id to use for the created event * @param inventoryItem the inventory item the event happened on * @param tagger the object that created the tag * @param tag the added/removed hashtag */ private void handleHashtagEvent(EventId eventId, InventoryItem inventoryItem, SobaObject tagger, String tag) { // Create the event Map<String, Object> eventContext = new HashMap<>(); if (eventId == EventId.HASHTAG_ADD) { eventContext.put("addedHashtag", tag); } else if (eventId == EventId.HASHTAG_DELETE) { eventContext.put("deletedHashtag", tag); } Event event = eventService.createEvent(eventId, inventoryItem, eventContext); // Create the message messageService.sendAccountMessage(event, tagger, inventoryItem.getConnection(), new Date().getTime(), MessageType.INVENTORY_ITEM, inventoryItem.getHashtags(), null); } private Map<String, InventoryItem> getInventoryItemMap(Connection connection) { List<InventoryItem> inventoryItems = getInventoryItems(connection); Map<String, InventoryItem> inventoryItemMap = new HashMap<>(); for (InventoryItem inventoryItem : inventoryItems) { inventoryItemMap.put(inventoryItem.getExternalId(), inventoryItem); } return inventoryItemMap; } private void pullEC2CloudWatchMetrics(Connection connection) throws ConnectionNotFoundException, InvalidCredentialsException { // Right now our CloudWatch usage is pretty specific in that we're not exactly pulling all available AWS EC2 // CloudWatch metrics and instead of relying on specific units for each metric name. Eventually we could/should // just pull down everything available and go from there. if (ec2CloudWatchMetricNames == null) { ec2CloudWatchMetricNames = new HashMap<>(); ec2CloudWatchMetricNames.put(EC2Constants.MetricName.CPU_UTILIZATION, Unit.PERCENT); ec2CloudWatchMetricNames.put(EC2Constants.MetricName.DISK_READ_BYTES, Unit.BYTES); ec2CloudWatchMetricNames.put(EC2Constants.MetricName.DISK_READ_OPS, Unit.COUNT); ec2CloudWatchMetricNames.put(EC2Constants.MetricName.DISK_WRITE_BYTES, Unit.BYTES); ec2CloudWatchMetricNames.put(EC2Constants.MetricName.DISK_WRITE_OPS, Unit.COUNT); ec2CloudWatchMetricNames.put(EC2Constants.MetricName.NETWORK_IN, Unit.BYTES); ec2CloudWatchMetricNames.put(EC2Constants.MetricName.NETWORK_OUT, Unit.BYTES); } try (RestContext<CloudWatchApi, CloudWatchAsyncApi> context = new AWSClient(connection) .getCloudWatchServiceContext()) { CloudWatchApi cloudWatchClient = context.getApi(); List<InventoryItem> inventoryItems = getInventoryItems(connection); String metricNamespace = Namespaces.EC2; Calendar cal = Calendar.getInstance(); Date endTime = new Date(); Date startTime; cal.add(Calendar.MINUTE, -30); startTime = cal.getTime(); for (InventoryItem inventoryItem : inventoryItems) { if (!inventoryItem.getType().equals(Constants.COMPUTE_INSTANCE_TYPE)) { continue; } Map<String, JSONObject> metrics = new HashMap<>(); String nodeId = inventoryItem.getExternalId(); Location region = getLocationByScope(inventoryItem, LocationScope.REGION); if (region == null) { continue; } String regionId = region.getId(); Dimension dimension = new Dimension(EC2Constants.Dimension.INSTANCE_ID, nodeId); for (Map.Entry<String, Unit> ec2MetricEntry : ec2CloudWatchMetricNames.entrySet()) { String metricName = ec2MetricEntry.getKey(); Unit metricUnit = ec2MetricEntry.getValue(); MetricApi metricClient = cloudWatchClient.getMetricApiForRegion(regionId); GetMetricStatistics requestOptions = GetMetricStatistics.builder().namespace(metricNamespace) .metricName(metricName).dimension(dimension).period(60) .statistics(ec2CloudWatchStatisticsSet).startTime(startTime).endTime(endTime) .unit(metricUnit).build(); GetMetricStatisticsResponse response = metricClient.getMetricStatistics(requestOptions); // Per Gustavo's code, we're only adding the last metric if (response != null && response.size() > 0) { metrics.put(metricName, JSONObject.fromObject(response.iterator().next())); } } if (!metrics.isEmpty()) { Map<String, Object> eventContext = new HashMap<>(); eventContext.put("payload", metrics); eventContext.put("isAgentActivity", false); eventService.createEvent(EventId.ACTIVITY, inventoryItem, eventContext); } } } } private void pullGitHubActivity(Connection connection) throws ConnectionNotFoundException, InvalidCredentialsException, IOException { GitHubClient client = (GitHubClient) getClient(connection); Map<String, InventoryItem> inventoryItemMap = getInventoryItemMap(connection); List<JSONObject> feedEntries = client.getActivity(inventoryItemMap.keySet()); Date lastActivityPoll = connection.getLastActivityPollDate(); Date lastActivity = lastActivityPoll; try { for (JSONObject entry : feedEntries) { String projectKey = entry.getJSONObject("repo").getString("name"); InventoryItem inventoryItem = inventoryItemMap.get(projectKey); if (inventoryItem == null) { continue; } Date pubDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'") .parse(entry.getString("created_at")); // Only create messages newer than the last activity poll date if (pubDate.before(lastActivityPoll)) { continue; } if (pubDate.after(lastActivity)) { lastActivity = pubDate; } Map<String, Object> activityParts = client.getPartsForActivity(inventoryItem, entry); // This can happen for unknown events which we log if (activityParts == null) { // We have ran into a GitHub activity we do not know how to handle. Log the issue with as much // detail as possible. String entryAsJSON = entry.toString(); logger.error("Unable to parse GitHub activity to create activity message: " + entryAsJSON); // Submit a bug report so we are aware of it. emailService.sendBugReport(Constants.NODEABLE_SUPER_USERNAME, Constants.NODEABLE_SUPER_ACCOUNT_NAME, "Unable to handle GitHub activity", "There was a GitHub activity that we currently do not handle.", entryAsJSON); // Should we create some specialized error message in the stream instead? // Move on to the next activity entry continue; } Map<String, Object> eventContext = new HashMap<>(); eventContext.put("activityPubDate", pubDate); eventContext.put("activityTitle", MessageUtils.cleanEntry((String) activityParts.get("title"))); eventContext.put("activityContent", MessageUtils.cleanEntry((String) activityParts.get("content"))); eventContext.put("activityHashtags", activityParts.get("hashtags")); eventContext.put("payload", entry.toString()); // Create the event stream entry Event event = eventService.createEvent(EventId.ACTIVITY, inventoryItem, eventContext); messageService.sendAccountMessage(event, inventoryItem, connection, pubDate.getTime(), MessageType.ACTIVITY, activityParts.get("hashtags") != null ? (Set<String>) activityParts.get("hashtags") : null, null); } } catch (Exception e) { logger.error("Unknown exception occurred while pulling GitHub activity for connection [" + connection.getId() + "]: " + e, e); } finally { // Update the connection's last polling time connection.setLastActivityPollDate(new Date(lastActivity.getTime() + 1)); try { connectionService.updateConnection(connection, true); } catch (Exception e) { // This is a silent update to only update the last polling time so this should never throw an exception } } } private void pullGoogleAnalyticsActivity(Connection connection) throws ConnectionNotFoundException, InvalidCredentialsException, IOException { GoogleAnalyticsClient client = (GoogleAnalyticsClient) getClient(connection); Map<String, InventoryItem> inventoryItemMap = getInventoryItemMap(connection); List<JSONObject> profileMetricEntries = client.getAllProfileMetrics(inventoryItemMap.keySet()); Date lastActivityPoll = connection.getLastActivityPollDate(); Date lastActivity = lastActivityPoll; try { for (JSONObject entry : profileMetricEntries) { if (entry == null) { continue; } String profileId = entry.getString("id"); InventoryItem inventoryItem = inventoryItemMap.get(profileId); if (inventoryItem == null) { continue; } JSONArray metrics = entry.getJSONArray("metrics"); for (Object obj : metrics) { JSONObject metric = (JSONObject) obj; Map<String, Object> eventContext = new HashMap<>(); eventContext.put("activityTitle", String.format("%s on %s is at %s", metric.getString("metric"), inventoryItem.getAlias(), metric.get("data"))); JSONArray jsonHashtags = metric.getJSONArray("hashtags"); Set<String> hashtags = new HashSet<String>(jsonHashtags); eventContext.put("activityHashtags", hashtags); eventContext.put("activityPayload", metric); /* need to fix this */ // Create the event stream entry Event event = eventService.createEvent(EventId.ACTIVITY, inventoryItem, eventContext); messageService.sendAccountMessage(event, inventoryItem, connection, System.currentTimeMillis(), MessageType.ACTIVITY, eventContext.get("activityHashtags") != null ? (Set<String>) eventContext.get("activityHashtags") : null, null); } } } catch (Exception e) { logger.error("Unknown exception occurred while pulling Google Analytics activity for connection [" + connection.getId() + "]: " + e, e); } finally { // Update the connection's last polling time connection.setLastActivityPollDate(new Date(lastActivity.getTime() + 1)); try { connectionService.updateConnection(connection, true); } catch (Exception e) { // This is a silent update to only update the last polling time so this should never throw an exception } } } private void pullJiraActivity(Connection connection) throws ConnectionNotFoundException, InvalidCredentialsException, IOException { JiraClient client = (JiraClient) getClient(connection); Map<String, InventoryItem> inventoryItemMap = getInventoryItemMap(connection); Date lastActivityPoll = connection.getLastActivityPollDate(); Date lastActivity = lastActivityPoll; try { List<Entry> feedEntries = client.getActivity(inventoryItemMap.keySet()); if (feedEntries == null) { return; } for (Entry entry : feedEntries) { // To map project activity in Jira to a Nodeable ProjectHostingInventoryItem, we have // to do some magic. Said magic is below. Element activityObject = entry .getFirstChild(new QName("http://activitystrea.ms/spec/1.0/", "object", "activity")); String projectKey = client.getProjectKeyOfEntry(activityObject, inventoryItemMap.keySet()); if (projectKey == null) { // If the projectKey is null here, this means we've gotten activity for a project we're monitoring // but we were unable to map said activity to the project in question. This is a known issue and // typically only seen in non-hosted Jira environments where people link their Jira project to other // Atlassian products but do not use the same key for the Jira project and the project in the other // Atlassian application. (SOBA-1193) Let's go ahead and log it so we do not forget but this is a // known issue and should not become a ZenDesk ticket. logger.error("Project key for Jira activity was unable to be found, possibly related to " + "SOBA-1193: " + entry.toString().substring(0, 140)); // Move on to the next activity entry continue; } InventoryItem inventoryItem = inventoryItemMap.get(projectKey); // This can happen if the activity is from a project, or Jira Studio product, not associated with a // project in our inventory system. (A good example of this is wiki changes. Each Jira Studio project // gets its own wiki but you can create new wiki spaces that are not associated with a Jira Studio // project and will end up without an inventory item in our system.) if (inventoryItem == null) { logger.error( "Project with key of " + projectKey + " did not correspond with an inventory item, " + "possibley related to SOBA-1193: " + entry.toString().substring(0, 140)); // Move on to the next activity entry continue; } Date pubDate = entry.getPublished(); // Only create messages newer than the last activity poll date if (pubDate.before(lastActivityPoll)) { continue; } if (pubDate.after(lastActivity)) { lastActivity = pubDate; } Map<String, Object> activityParts = client.getPartsForActivity(inventoryItem, entry); // This can happen for unknown events which we log if (activityParts == null) { // We have ran into a Jira activity we do not know how to handle. Log the issue with as much // detail as possible. String entryAsJSON = entry.toString(); logger.error("Unable to parse Jira activity to create activity message: " + entryAsJSON); // Submit a but report so we are aware of it. emailService.sendBugReport(Constants.NODEABLE_SUPER_USERNAME, Constants.NODEABLE_SUPER_ACCOUNT_NAME, "Unable to handle Jira activity", "There was a Jira activity that we currently do not handle.", entryAsJSON); // Should we create some specialized error message in the stream instead? // Move on to the next activity entry continue; } Map<String, Object> eventContext = new HashMap<>(); eventContext.put("activityPubDate", pubDate); eventContext.put("activityTitle", MessageUtils.cleanEntry((String) activityParts.get("title"))); eventContext.put("activityContent", MessageUtils.cleanEntry((String) activityParts.get("content"))); eventContext.put("activityHashtags", activityParts.get("hashtags")); eventContext.put("payload", JSONUtils.xmlToJSON(entry.toString()).toString()); // Create the event stream entry Event event = eventService.createEvent(EventId.ACTIVITY, inventoryItem, eventContext); JiraActivityDetails details = getJiraActivityDetailsFromActivityParts(activityParts); messageService.sendAccountMessage(event, inventoryItem, connection, pubDate.getTime(), MessageType.ACTIVITY, activityParts.get("hashtags") != null ? (Set<String>) activityParts.get("hashtags") : null, details); } } catch (Exception e) { logger.error("Unknown exception occurred while pulling Jira activity for connection [" + connection.getId() + "]: " + e, e); } finally { // Update the connection's last polling time connection.setLastActivityPollDate(new Date(lastActivity.getTime() + 1)); try { connectionService.updateConnection(connection, true); } catch (Exception e) { // This is a silent update to only update the last polling time so this should never throw an exception } } } private void pullFeedActivity(Connection connection) throws ConnectionNotFoundException, InvalidCredentialsException, IOException { Date lastActivityPoll = connection.getLastActivityPollDate(); Date lastActivity = lastActivityPoll; if (lastActivityPoll != null) { logger.debug("Creating feed messages for messages newer than (" + lastActivityPoll + ") for [" + connection.getId() + "]: " + connection.getAlias()); } else { logger.debug("Creating feed messages for all messages [" + connection.getId() + "]: " + connection.getAlias()); } try (XmlReader xmlReader = new XmlReader(URI.create(connection.getUrl()).toURL())) { SyndFeed rssFeed = new SyndFeedInput().build(xmlReader); List feedEntries = rssFeed.getEntries(); Collections.sort(feedEntries, new Comparator<Object>() { @Override public int compare(Object first, Object second) { SyndEntry firstEntry = (SyndEntry) first; SyndEntry secondEntry = (SyndEntry) second; Date firstDate = firstEntry.getPublishedDate() != null ? firstEntry.getPublishedDate() : new Date(); Date secondDate = secondEntry.getPublishedDate() != null ? secondEntry.getPublishedDate() : new Date(); return firstDate.compareTo(secondDate); } }); for (Object rawEntry : feedEntries) { SyndEntry entry = (SyndEntry) rawEntry; //use published date if it exists... otherwise don't process the message as it is an update //this skips feed messages from feeds that don't include a publishedDate Date pubDate = entry.getPublishedDate(); if (pubDate == null || pubDate.before(lastActivityPoll)) { continue; } lastActivity = pubDate.after(lastActivity) ? pubDate : lastActivity; Map<String, Object> eventContext = new HashMap<>(); String messageBodyAsJson = determineMessageBodyAsJsonFromSyndEntry(entry); eventContext.put("activityPubDate", pubDate); eventContext.put("activityTitle", entry.getTitle()); eventContext.put("payload", messageBodyAsJson); Event event = eventService.createEvent(EventId.ACTIVITY, connection, eventContext); FeedEntryDetails details = new FeedEntryDetails.Builder().url(entry.getUri()) .title(entry.getTitle()) .description(entry.getDescription() != null ? entry.getDescription().getValue() : null) .publishedDate(pubDate).build(); // Create a new message to be delivered to inboxes messageService.sendActivityMessage(event, connection, pubDate.getTime(), details); } } catch (IOException e) { logger.error(String.format("Error opening the connection %s for feed %s. Returned error: %s", connection.getId(), connection.getUrl(), e.getMessage())); } catch (Exception e) { logger.error( String.format("Unable to process messages for connection %s with feed %s. Returned error: %s", connection.getId(), connection.getUrl(), e.getMessage())); } finally { // Update the connection's last polling timeconnection.setLastActivityPollDate(new Date(lastActivity.getTime() + 1)); try { connectionService.updateConnection(connection, true); } catch (Exception e) { // This is a silent update to only update the last polling time so this should never throw an exception } } } private void pullPingdomActivity(Connection connection) throws ConnectionNotFoundException, InvalidCredentialsException, IOException { PingdomClient client = (PingdomClient) getClient(connection); Map<String, InventoryItem> inventoryItemMap = getInventoryItemMap(connection); Date lastActivityPoll = connection.getLastActivityPollDate(); if (lastActivityPoll != null) { logger.debug("Creating Pingdom messages for messages newer than (" + lastActivityPoll + ") for [" + connection.getId() + "]: " + connection.getAlias()); } else { logger.debug("Creating Pingdom messages for all messages [" + connection.getId() + "]: " + connection.getAlias()); } try { List<JSONObject> jsonInventoryList = client.checks(); for (Iterator<JSONObject> i = jsonInventoryList.iterator(); i.hasNext();) { JSONObject jsonInventory = i.next(); InventoryItem inventoryItem = inventoryItemMap.get(jsonInventory.getString("id")); if (inventoryItem == null) { continue; } // the connection may not have been tested yet from Pingdom if (!jsonInventory.containsKey("lasttesttime")) { continue; } Date lastTestTime = new DateTime().withMillis(0).plusSeconds(jsonInventory.getInt("lasttesttime")) .toDate(); if (lastActivityPoll != null && lastActivityPoll.after(lastTestTime)) { continue; } lastActivityPoll = lastTestTime.after(lastActivityPoll) ? lastTestTime : lastActivityPoll; Map<String, Object> eventContext = new HashMap<>(); eventContext.put("activityPubDate", lastTestTime); eventContext.put("activityTitle", String.format("Service check %s (%s) has a response time of %dms.", jsonInventory.getString("name"), jsonInventory.getString("type").toUpperCase(), jsonInventory.getInt("lastresponsetime"))); eventContext.put("payload", jsonInventory); if (jsonInventory.containsKey("lastresponsetime")) { eventContext.put("lastResponseTime", jsonInventory.getInt("lastresponsetime")); } Event event = eventService.createEvent(EventId.ACTIVITY, inventoryItem, eventContext); PingdomEntryDetails details = new PingdomEntryDetails.Builder().lastErrorTime( jsonInventory.containsKey("lasterrortime") ? jsonInventory.getInt("lasterrortime") : 0) .lastResponseTime(jsonInventory.containsKey("lastresponsetime") ? jsonInventory.getInt("lastresponsetime") : 0) .lastTestTime(jsonInventory.getInt("lasttesttime")) .checkCreated(jsonInventory.getInt("created")) .resolution(jsonInventory.getInt("resolution")).status(jsonInventory.getString("status")) .build(); // Create a new message to be delivered to inboxes messageService.sendAccountMessage(event, inventoryItem, connection, lastTestTime.getTime(), MessageType.ACTIVITY, inventoryItem.getHashtags(), details); } // Update the connection's last polling time connection.setLastActivityPollDate(new Date(lastActivityPoll.getTime() + 1)); connectionService.updateConnection(connection, true); } catch (IOException e) { logger.error(String.format("Error opening the connection %s for feed %s. Returned error: %s", connection.getId(), connection.getUrl(), e.getMessage())); } catch (Exception e) { logger.error( String.format("Unable to process messages for connection %s with feed %s. Returned error: %s", connection.getId(), connection.getUrl(), e.getMessage())); } } private void pullTwitterActivity(Connection connection) { TwitterClient client = (TwitterClient) getClient(connection); try { JSONObject profile = client.getLoggedInProfile(); if (profile == null) { logger.error("User's profile for Twitter connection %s came back null.", connection.getId()); return; } Map<String, Object> eventContext = new HashMap<>(); int favoritesCount = profile.containsKey("favourites_count") ? profile.getInt("favourites_count") : 0; int followersCount = profile.containsKey("followers_count") ? profile.getInt("followers_count") : 0; int friendsCount = profile.containsKey("friends_count") ? profile.getInt("friends_count") : 0; int listedCount = profile.containsKey("listed_count") ? profile.getInt("listed_count") : 0; int statusesCount = profile.containsKey("statuses_count") ? profile.getInt("statuses_count") : 0; String screenName = profile.containsKey("screen_name") ? profile.getString("screen_name") : "unknown"; String name = profile.containsKey("name") ? profile.getString("name") : "Unknown"; // Create a "title" for the activity (Shouldn't be here but without refactoring message transformation, it // is what it is.) String activityTitle = String.format("Twitter user stats for %s (%s)", screenName, name); String activityContent = String.format( "Following %d users, has %d followers, has tweeted %d times, has " + "created %d favorites and has been added to %d lists.", friendsCount, followersCount, statusesCount, favoritesCount, listedCount); eventContext.put("activityTitle", activityTitle); eventContext.put("activityContent", activityContent); eventContext.put("payload", profile); Event event = eventService.createEvent(EventId.ACTIVITY, connection, eventContext); TwitterActivityDetails details = new TwitterActivityDetails.Builder().favorites(favoritesCount) .followers(followersCount).friends(friendsCount).profile(profile).statuses(statusesCount) .build(); messageService.sendActivityMessage(event, connection, new Date().getTime(), details); } catch (Exception e) { logger.error( String.format("Error getting the user's profile for Twitter connection %s. Returned error: %s", connection.getId(), e.getMessage())); } } /** * Extracts a message body from a Rome SyndEntry object. This handles calculating a message body from SyndEntry * objects that might either represent an entry in an RSS feed or an Atompub feed. * * @param entry SyndEntry to extract a message body from * @return JSON mapping of the description if it exists, or the first content element if there are contents. * Otherwise an empty string is mapped to Json * @throws IOException when the SyndEntry fields can't be read or parsed to JSON. */ private String determineMessageBodyAsJsonFromSyndEntry(SyndEntry entry) throws IOException { String body = ""; //default Body ObjectMapper om = new ObjectMapper(); SyndContent desc = entry.getDescription(); if (desc != null) { body = om.writeValueAsString(desc); } else if (entry.getContents().size() > 0) { body = om.writeValueAsString(entry.getContents().get(0)); //grab the first content item and use as body } return body; } /** * Creates a JiraActivityDetails object from the activityParts Map. Specifically, this looks up the rawContent * part of Jira activity and stores in the html field of JiraActivities for display in rich messages. Since it is * possible for jira messages to be extremely long (for instance, a commit message of a branch * that copies several thousand files will result in an html payload of tens of thousand of characters), this method * will return null so that activity details are not kept and clients will resort to non-richly formatted content. * * @param activityParts activityParts created from polling a jira instance * @return JiraActivityDetails with an html field if activityParts is not null, its "rawContent" key has a non-null * value, and the value of "rawContent" does not exceed a predefined number of characters. */ private JiraActivityDetails getJiraActivityDetailsFromActivityParts(Map<String, Object> activityParts) { if (activityParts == null) { return null; } String rawContent = (String) activityParts.get("rawContent"); if (rawContent == null || rawContent.length() > Constants.MAX_MESSAGE_LENGTH) { return null; } return new JiraActivityDetails.Builder().html((rawContent)).build(); } private ExternalIntegrationClient getClient(Connection connection) { ExternalIntegrationClient client; client = externalClientCache.getIfPresent(connection.getId()); if (client == null) { client = connectionProviderFactory .externalIntegrationConnectionProviderFromId(connection.getProviderId()).getClient(connection); externalClientCache.put(connection.getId(), client); } return client; } }