Java tutorial
/* * Copyright 2014-2015 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of * the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. */ package co.cask.cdap.logging.context; import co.cask.cdap.common.conf.Constants; import co.cask.cdap.common.logging.ApplicationLoggingContext; import co.cask.cdap.common.logging.ComponentLoggingContext; import co.cask.cdap.common.logging.LoggingContext; import co.cask.cdap.common.logging.NamespaceLoggingContext; import co.cask.cdap.common.logging.ServiceLoggingContext; import co.cask.cdap.common.logging.SystemLoggingContext; import co.cask.cdap.logging.filter.AndFilter; import co.cask.cdap.logging.filter.Filter; import co.cask.cdap.logging.filter.MdcExpression; import co.cask.cdap.logging.filter.OrFilter; import co.cask.cdap.proto.ProgramType; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Map; import javax.annotation.Nullable; /** * Helper class for LoggingContext objects. */ public final class LoggingContextHelper { private static final String ACCOUNT_ID = ".accountId"; private static final Logger LOG = LoggerFactory.getLogger(LoggingContext.class); private static final Map<String, String> LOG_TAG_TO_METRICS_TAG_MAP = ImmutableMap.<String, String>builder() .put(FlowletLoggingContext.TAG_FLOWLET_ID, Constants.Metrics.Tag.FLOWLET) .put(WorkflowLoggingContext.TAG_WORKFLOW_ID, Constants.Metrics.Tag.WORKFLOW) .put(MapReduceLoggingContext.TAG_MAP_REDUCE_JOB_ID, Constants.Metrics.Tag.MAPREDUCE) .put(SparkLoggingContext.TAG_SPARK_JOB_ID, Constants.Metrics.Tag.SPARK) .put(UserServiceLoggingContext.TAG_USERSERVICE_ID, Constants.Metrics.Tag.HANDLER) .put(WorkerLoggingContext.TAG_WORKER_ID, Constants.Metrics.Tag.WORKER).build(); private LoggingContextHelper() { } public static String getNamespacedBaseDir(String logBaseDir, String logPartition) { Preconditions.checkArgument(logBaseDir != null, "Log Base dir cannot be null"); Preconditions.checkArgument(logPartition != null, "Log partition cannot be null"); String[] partitions = logPartition.split(":"); Preconditions.checkArgument(partitions.length == 3, "Expected log partition to be in the format <ns>:<entity>:<sub-entity>"); // don't care about the app or the program, only need the namespace GenericLoggingContext loggingContext = new GenericLoggingContext(partitions[0], partitions[1], partitions[2]); return loggingContext.getNamespacedLogBaseDir(logBaseDir); } public static LoggingContext getLoggingContext(Map<String, String> tags) { // Tags are empty, cannot determine logging context. if (tags == null || tags.isEmpty()) { throw new IllegalArgumentException("Tags are empty, cannot determine logging context"); } String namespaceId = tags.get(NamespaceLoggingContext.TAG_NAMESPACE_ID); String applicationId = tags.get(ApplicationLoggingContext.TAG_APPLICATION_ID); String systemId = tags.get(SystemLoggingContext.TAG_SYSTEM_ID); String componentId = tags.get(ComponentLoggingContext.TAG_COMPONENT_ID); // No namespace id or application id present. if (namespaceId == null || applicationId == null) { if (systemId == null || componentId == null) { throw new IllegalArgumentException("No namespace/application or system/component id present"); } } if (tags.containsKey(FlowletLoggingContext.TAG_FLOW_ID)) { if (!tags.containsKey(FlowletLoggingContext.TAG_FLOWLET_ID)) { return null; } return new FlowletLoggingContext(namespaceId, applicationId, tags.get(FlowletLoggingContext.TAG_FLOW_ID), tags.get(FlowletLoggingContext.TAG_FLOWLET_ID), tags.get(ApplicationLoggingContext.TAG_RUNID_ID), tags.get(ApplicationLoggingContext.TAG_INSTANCE_ID)); } else if (tags.containsKey(WorkflowLoggingContext.TAG_WORKFLOW_ID)) { return new WorkflowLoggingContext(namespaceId, applicationId, tags.get(WorkflowLoggingContext.TAG_WORKFLOW_ID), tags.get(ApplicationLoggingContext.TAG_RUNID_ID), tags.get(ApplicationLoggingContext.TAG_ADAPTER_ID)); } else if (tags.containsKey(MapReduceLoggingContext.TAG_MAP_REDUCE_JOB_ID)) { return new MapReduceLoggingContext(namespaceId, applicationId, tags.get(MapReduceLoggingContext.TAG_MAP_REDUCE_JOB_ID), tags.get(ApplicationLoggingContext.TAG_RUNID_ID), tags.get(ApplicationLoggingContext.TAG_ADAPTER_ID)); } else if (tags.containsKey(SparkLoggingContext.TAG_SPARK_JOB_ID)) { return new SparkLoggingContext(namespaceId, applicationId, tags.get(SparkLoggingContext.TAG_SPARK_JOB_ID), tags.get(ApplicationLoggingContext.TAG_RUNID_ID)); } else if (tags.containsKey(UserServiceLoggingContext.TAG_USERSERVICE_ID)) { if (!tags.containsKey(UserServiceLoggingContext.TAG_HANDLER_ID)) { return null; } return new UserServiceLoggingContext(namespaceId, applicationId, tags.get(UserServiceLoggingContext.TAG_USERSERVICE_ID), tags.get(UserServiceLoggingContext.TAG_HANDLER_ID), tags.get(ApplicationLoggingContext.TAG_RUNID_ID), tags.get(ApplicationLoggingContext.TAG_INSTANCE_ID)); } else if (tags.containsKey(ServiceLoggingContext.TAG_SERVICE_ID)) { return new ServiceLoggingContext(systemId, componentId, tags.get(ServiceLoggingContext.TAG_SERVICE_ID)); } else if (tags.containsKey(WorkerLoggingContext.TAG_WORKER_ID)) { return new WorkerLoggingContext(namespaceId, applicationId, tags.get(WorkerLoggingContext.TAG_WORKER_ID), tags.get(ApplicationLoggingContext.TAG_RUNID_ID), tags.get(ApplicationLoggingContext.TAG_INSTANCE_ID), tags.get(ApplicationLoggingContext.TAG_ADAPTER_ID)); } throw new IllegalArgumentException("Unsupported logging context"); } public static LoggingContext getLoggingContext(String systemId, String componentId, String serviceId) { return new ServiceLoggingContext(systemId, componentId, serviceId); } public static LoggingContext getLoggingContext(String namespaceId, String applicationId, String entityId, ProgramType programType) { return getLoggingContext(namespaceId, applicationId, entityId, programType, null); } public static LoggingContext getLoggingContext(String namespaceId, String applicationId, String entityId, ProgramType programType, @Nullable String adapterName) { return getLoggingContext(namespaceId, applicationId, entityId, programType, null, adapterName); } public static LoggingContext getLoggingContextWithRunId(String namespaceId, String applicationId, String entityId, ProgramType programType, String runId, @Nullable String adapterName) { return getLoggingContext(namespaceId, applicationId, entityId, programType, runId, adapterName); } public static LoggingContext getLoggingContext(String namespaceId, String applicationId, String entityId, ProgramType programType, @Nullable String runId, @Nullable String adapterName) { switch (programType) { case FLOW: return new FlowletLoggingContext(namespaceId, applicationId, entityId, "", runId, null); case WORKFLOW: return new WorkflowLoggingContext(namespaceId, applicationId, entityId, runId, adapterName); case MAPREDUCE: return new MapReduceLoggingContext(namespaceId, applicationId, entityId, runId, adapterName); case SPARK: return new SparkLoggingContext(namespaceId, applicationId, entityId, runId); case SERVICE: return new UserServiceLoggingContext(namespaceId, applicationId, entityId, "", runId, null); case WORKER: return new WorkerLoggingContext(namespaceId, applicationId, entityId, runId, null, adapterName); default: throw new IllegalArgumentException( String.format("Illegal entity type for logging context: %s", programType)); } } public static Filter createFilter(LoggingContext loggingContext) { if (loggingContext instanceof ServiceLoggingContext) { String systemId = loggingContext.getSystemTagsMap().get(ServiceLoggingContext.TAG_SYSTEM_ID).getValue(); String componentId = loggingContext.getSystemTagsMap().get(ServiceLoggingContext.TAG_COMPONENT_ID) .getValue(); String tagName = ServiceLoggingContext.TAG_SERVICE_ID; String entityId = loggingContext.getSystemTagsMap().get(ServiceLoggingContext.TAG_SERVICE_ID) .getValue(); return new AndFilter(ImmutableList.of(new MdcExpression(ServiceLoggingContext.TAG_SYSTEM_ID, systemId), new MdcExpression(ServiceLoggingContext.TAG_COMPONENT_ID, componentId), new MdcExpression(tagName, entityId))); } else { String namespaceId = loggingContext.getSystemTagsMap().get(ApplicationLoggingContext.TAG_NAMESPACE_ID) .getValue(); String applId = loggingContext.getSystemTagsMap().get(ApplicationLoggingContext.TAG_APPLICATION_ID) .getValue(); LoggingContext.SystemTag entityTag = getEntityId(loggingContext); ImmutableList.Builder<Filter> filterBuilder = ImmutableList.builder(); // For backward compatibility: The old logs before namespace have .accountId and developer as value so we don't // want them to get filtered out if they belong to this application and entity OrFilter namespaceFilter = new OrFilter( ImmutableList.of(new MdcExpression(NamespaceLoggingContext.TAG_NAMESPACE_ID, namespaceId), new MdcExpression(ACCOUNT_ID, Constants.DEVELOPER_ACCOUNT))); filterBuilder.add(namespaceFilter); filterBuilder.add(new MdcExpression(ApplicationLoggingContext.TAG_APPLICATION_ID, applId)); filterBuilder.add(new MdcExpression(entityTag.getName(), entityTag.getValue())); // Add runid filter if required LoggingContext.SystemTag runId = loggingContext.getSystemTagsMap() .get(ApplicationLoggingContext.TAG_RUNID_ID); if (runId != null && runId.getValue() != null) { filterBuilder.add(new MdcExpression(ApplicationLoggingContext.TAG_RUNID_ID, runId.getValue())); } // Add adapter filter if required if (loggingContext.getSystemTagsMap().containsKey(ApplicationLoggingContext.TAG_ADAPTER_ID)) { String adapterName = loggingContext.getSystemTagsMap().get(ApplicationLoggingContext.TAG_ADAPTER_ID) .getValue(); filterBuilder.add(new MdcExpression(ApplicationLoggingContext.TAG_ADAPTER_ID, adapterName)); } return new AndFilter(filterBuilder.build()); } } public static LoggingContext.SystemTag getEntityId(LoggingContext loggingContext) { final String tagName; if (loggingContext instanceof FlowletLoggingContext) { tagName = FlowletLoggingContext.TAG_FLOW_ID; } else if (loggingContext instanceof WorkflowLoggingContext) { tagName = WorkflowLoggingContext.TAG_WORKFLOW_ID; } else if (loggingContext instanceof MapReduceLoggingContext) { tagName = MapReduceLoggingContext.TAG_MAP_REDUCE_JOB_ID; } else if (loggingContext instanceof SparkLoggingContext) { tagName = SparkLoggingContext.TAG_SPARK_JOB_ID; } else if (loggingContext instanceof UserServiceLoggingContext) { tagName = UserServiceLoggingContext.TAG_USERSERVICE_ID; } else if (loggingContext instanceof WorkerLoggingContext) { tagName = WorkerLoggingContext.TAG_WORKER_ID; } else { throw new IllegalArgumentException(String.format("Invalid logging context: %s", loggingContext)); } final String entityId = loggingContext.getSystemTagsMap().get(tagName).getValue(); return new LoggingContext.SystemTag() { @Override public String getName() { return tagName; } @Override public String getValue() { return entityId; } }; } public static Map<String, String> getMetricsTags(LoggingContext context) throws IllegalArgumentException { if (context instanceof ServiceLoggingContext) { return getMetricsTagsFromSystemContext((ServiceLoggingContext) context); } else { return getMetricsTagsFromLoggingContext(context); } } private static Map<String, String> getMetricsTagsFromLoggingContext(LoggingContext context) { ImmutableMap.Builder<String, String> builder = ImmutableMap.builder(); Map<String, LoggingContext.SystemTag> loggingTags = context.getSystemTagsMap(); String namespace = getValueFromTag(loggingTags.get(NamespaceLoggingContext.TAG_NAMESPACE_ID)); if (namespace == null || namespace.isEmpty()) { throw new IllegalArgumentException("Cannot find namespace in logging context"); } builder.put(Constants.Metrics.Tag.NAMESPACE, namespace); String applicationId = getValueFromTag(loggingTags.get(ApplicationLoggingContext.TAG_APPLICATION_ID)); String adapterId = getValueFromTag(loggingTags.get(ApplicationLoggingContext.TAG_ADAPTER_ID)); // Must be an application or adapter if (applicationId == null && adapterId == null) { throw new IllegalArgumentException("Missing application or adapter id"); } if (applicationId != null) { builder.put(Constants.Metrics.Tag.APP, applicationId); LoggingContext.SystemTag entityId = getEntityId(context); String entityName = getMetricsTagNameFromLoggingContext(entityId); if (entityName != null) { builder.put(entityName, entityId.getValue()); } } else { builder.put(Constants.Metrics.Tag.ADAPTER, adapterId); } return builder.build(); } private static String getMetricsTagNameFromLoggingContext(LoggingContext.SystemTag tag) { return LOG_TAG_TO_METRICS_TAG_MAP.get(tag.getName()); } private static String getValueFromTag(LoggingContext.SystemTag tag) { return tag == null ? null : tag.getValue(); } private static Map<String, String> getMetricsTagsFromSystemContext(ServiceLoggingContext context) { ImmutableMap.Builder<String, String> builder = ImmutableMap.builder(); builder.put(Constants.Metrics.Tag.NAMESPACE, Constants.SYSTEM_NAMESPACE); builder.put(Constants.Metrics.Tag.COMPONENT, context.getSystemTagsMap().get(ServiceLoggingContext.TAG_SERVICE_ID).getValue()); return builder.build(); } }