Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.gobblin.metrics; import java.io.IOException; import java.io.OutputStream; import java.net.URI; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.gobblin.metrics.reporter.FileFailureEventReporter; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.codahale.metrics.Counter; import com.codahale.metrics.Histogram; import com.codahale.metrics.JmxReporter; import com.codahale.metrics.Meter; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Timer; import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.base.Splitter; import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.io.Closer; import com.typesafe.config.Config; import org.apache.gobblin.configuration.ConfigurationKeys; import org.apache.gobblin.configuration.State; import org.apache.gobblin.metrics.graphite.GraphiteConnectionType; import org.apache.gobblin.metrics.graphite.GraphiteEventReporter; import org.apache.gobblin.metrics.graphite.GraphiteReporter; import org.apache.gobblin.metrics.influxdb.InfluxDBConnectionType; import org.apache.gobblin.metrics.influxdb.InfluxDBEventReporter; import org.apache.gobblin.metrics.influxdb.InfluxDBReporter; import org.apache.gobblin.metrics.reporter.OutputStreamEventReporter; import org.apache.gobblin.metrics.reporter.OutputStreamReporter; import org.apache.gobblin.metrics.reporter.ScheduledReporter; import org.apache.gobblin.password.PasswordManager; import org.apache.gobblin.util.PropertiesUtils; /** * A class that represents a set of metrics associated with a given name. * * @author Yinan Li */ public class GobblinMetrics { public static final String METRICS_STATE_CUSTOM_TAGS = "metrics.state.custom.tags"; protected static final GobblinMetricsRegistry GOBBLIN_METRICS_REGISTRY = GobblinMetricsRegistry.getInstance(); /** * Enumeration of metric types. */ public enum MetricType { COUNTER, METER, GAUGE } private static final Logger LOGGER = LoggerFactory.getLogger(GobblinMetrics.class); /** * Check whether metrics collection and reporting are enabled or not. * * @param properties Configuration properties * @return whether metrics collection and reporting are enabled */ public static boolean isEnabled(Properties properties) { return PropertiesUtils.getPropAsBoolean(properties, ConfigurationKeys.METRICS_ENABLED_KEY, ConfigurationKeys.DEFAULT_METRICS_ENABLED); } /** * Check whether metrics collection and reporting are enabled or not. * * @param state a {@link State} object containing configuration properties * @return whether metrics collection and reporting are enabled */ public static boolean isEnabled(State state) { return Boolean.valueOf( state.getProp(ConfigurationKeys.METRICS_ENABLED_KEY, ConfigurationKeys.DEFAULT_METRICS_ENABLED)); } /** * Check whether metrics collection and reporting are enabled or not. * * @param cfg a {@link State} object containing configuration properties * @return whether metrics collection and reporting are enabled */ public static boolean isEnabled(Config cfg) { return cfg.hasPath(ConfigurationKeys.METRICS_ENABLED_KEY) ? cfg.getBoolean(ConfigurationKeys.METRICS_ENABLED_KEY) : Boolean.parseBoolean(ConfigurationKeys.DEFAULT_METRICS_ENABLED); } /** * Get a {@link GobblinMetrics} instance with the given ID. * * @param id the given {@link GobblinMetrics} ID * @return a {@link GobblinMetrics} instance */ public static GobblinMetrics get(String id) { return get(id, null); } /** * Get a {@link GobblinMetrics} instance with the given ID and parent {@link MetricContext}. * * @param id the given {@link GobblinMetrics} ID * @param parentContext the given parent {@link MetricContext} * @return a {@link GobblinMetrics} instance */ public static GobblinMetrics get(String id, MetricContext parentContext) { return get(id, parentContext, Lists.<Tag<?>>newArrayList()); } /** * Get a {@link GobblinMetrics} instance with the given ID, parent {@link MetricContext}, * and list of {@link Tag}s. * * @param id the given {@link GobblinMetrics} ID * @param parentContext the given parent {@link MetricContext} * @param tags the given list of {@link Tag}s * @return a {@link GobblinMetrics} instance */ public static GobblinMetrics get(final String id, final MetricContext parentContext, final List<Tag<?>> tags) { return GOBBLIN_METRICS_REGISTRY.getOrDefault(id, new Callable<GobblinMetrics>() { @Override public GobblinMetrics call() throws Exception { return new GobblinMetrics(id, parentContext, tags); } }); } /** * Remove the {@link GobblinMetrics} instance associated with the given ID. * * @param id the given {@link GobblinMetrics} ID */ public static void remove(String id) { GOBBLIN_METRICS_REGISTRY.remove(id); } /** * Add a {@link List} of {@link Tag}s to a {@link org.apache.gobblin.configuration.State} with key {@link #METRICS_STATE_CUSTOM_TAGS}. * * <p> * {@link org.apache.gobblin.metrics.Tag}s under this key can later be parsed using the method {@link #getCustomTagsFromState}. * </p> * * @param state {@link org.apache.gobblin.configuration.State} state to add the tag to. * @param tags list of {@link Tag}s to add. */ public static void addCustomTagToState(State state, List<? extends Tag<?>> tags) { for (Tag<?> tag : tags) { state.appendToListProp(METRICS_STATE_CUSTOM_TAGS, tag.toString()); } } /** * Add a {@link Tag} to a {@link org.apache.gobblin.configuration.State} with key {@link #METRICS_STATE_CUSTOM_TAGS}. * * <p> * {@link org.apache.gobblin.metrics.Tag}s under this key can later be parsed using the method {@link #getCustomTagsFromState}. * </p> * * @param state {@link org.apache.gobblin.configuration.State} state to add the tag to. * @param tag {@link Tag} to add. */ public static void addCustomTagToState(State state, Tag<?> tag) { state.appendToListProp(METRICS_STATE_CUSTOM_TAGS, tag.toString()); } /** * Add {@link List} of {@link Tag}s to a {@link Properties} with key {@link #METRICS_STATE_CUSTOM_TAGS}. * <p> * Also see {@link #addCustomTagToState(State, Tag)} , {@link #addCustomTagToProperties(Properties, Tag)} * </p> * * <p> * The {@link Properties} passed can be used to build a {@link State}. * {@link org.apache.gobblin.metrics.Tag}s under this key can later be parsed using the method {@link #getCustomTagsFromState}. * </p> * * @param properties {@link Properties} to add the tag to. * @param tags list of {@link Tag}s to add. */ public static void addCustomTagsToProperties(Properties properties, List<Tag<?>> tags) { for (Tag<?> tag : tags) { addCustomTagToProperties(properties, tag); } } /** * Add a {@link Tag} to a {@link Properties} with key {@link #METRICS_STATE_CUSTOM_TAGS}. * Also see {@link #addCustomTagToState(State, Tag)} * * <p> * The {@link Properties} passed can be used to build a {@link State}. * {@link org.apache.gobblin.metrics.Tag}s under this key can later be parsed using the method {@link #getCustomTagsFromState}. * </p> * * @param properties {@link Properties} to add the tag to. * @param tag {@link Tag} to add. */ public static void addCustomTagToProperties(Properties properties, Tag<?> tag) { // Build a state wrapper to add custom tag to property State state = new State(properties); addCustomTagToState(state, tag); } /** * Parse custom {@link org.apache.gobblin.metrics.Tag}s from property {@link #METRICS_STATE_CUSTOM_TAGS} * in the input {@link org.apache.gobblin.configuration.State}. * @param state {@link org.apache.gobblin.configuration.State} possibly containing custom tags. * @return List of {@link org.apache.gobblin.metrics.Tag} parsed from input. */ public static List<Tag<?>> getCustomTagsFromState(State state) { List<Tag<?>> tags = Lists.newArrayList(); for (String tagKeyValue : state.getPropAsList(METRICS_STATE_CUSTOM_TAGS, "")) { Tag<?> tag = Tag.fromString(tagKeyValue); if (tag != null) { tags.add(tag); } } return tags; } protected final String id; protected final MetricContext metricContext; // Closer for closing the metric output stream protected final Closer codahaleReportersCloser = Closer.create(); // JMX metric reporter private Optional<JmxReporter> jmxReporter = Optional.absent(); // Custom metric reporters instantiated through reflection private final List<com.codahale.metrics.ScheduledReporter> codahaleScheduledReporters = Lists.newArrayList(); // A flag telling whether metric reporting has started or not private volatile boolean metricsReportingStarted = false; protected GobblinMetrics(String id, MetricContext parentContext, List<Tag<?>> tags) { this.id = id; this.metricContext = parentContext == null ? new MetricContext.Builder(id).addTags(tags).build() : parentContext.childBuilder(id).addTags(tags).build(); } /** * Get the wrapped {@link com.codahale.metrics.MetricRegistry} instance. * * @return wrapped {@link com.codahale.metrics.MetricRegistry} instance */ public MetricContext getMetricContext() { return this.metricContext; } /** * Get the ID of this {@link GobblinMetrics}. * * @return ID of this {@link GobblinMetrics} */ public String getId() { return this.id; } /** * Get the name of this {@link GobblinMetrics}. * * <p> * This method is currently equivalent to {@link #getId()}. * </p> * * @return name of this {@link GobblinMetrics} */ public String getName() { return this.id; } /** * Get a {@link Meter} with the given name prefix and suffixes. * * @param prefix the given name prefix * @param suffixes the given name suffixes * @return a {@link Meter} with the given name prefix and suffixes */ public Meter getMeter(String prefix, String... suffixes) { return this.metricContext.meter(MetricRegistry.name(prefix, suffixes)); } /** * Get a {@link Counter} with the given name prefix and suffixes. * * @param prefix the given name prefix * @param suffixes the given name suffixes * @return a {@link Counter} with the given name prefix and suffixes */ public Counter getCounter(String prefix, String... suffixes) { return this.metricContext.counter(MetricRegistry.name(prefix, suffixes)); } /** * Get a {@link Histogram} with the given name prefix and suffixes. * * @param prefix the given name prefix * @param suffixes the given name suffixes * @return a {@link Histogram} with the given name prefix and suffixes */ public Histogram getHistogram(String prefix, String... suffixes) { return this.metricContext.histogram(MetricRegistry.name(prefix, suffixes)); } /** * Get a {@link Timer} with the given name prefix and suffixes. * * @param prefix the given name prefix * @param suffixes the given name suffixes * @return a {@link Timer} with the given name prefix and suffixes */ public Timer getTimer(String prefix, String... suffixes) { return this.metricContext.timer(MetricRegistry.name(prefix, suffixes)); } /** * Starts metric reporting and appends the given metrics file suffix to the current value of * {@link ConfigurationKeys#METRICS_FILE_SUFFIX}. */ public void startMetricReportingWithFileSuffix(State state, String metricsFileSuffix) { Properties metricsReportingProps = new Properties(); metricsReportingProps.putAll(state.getProperties()); String oldMetricsFileSuffix = state.getProp(ConfigurationKeys.METRICS_FILE_SUFFIX, ConfigurationKeys.DEFAULT_METRICS_FILE_SUFFIX); if (Strings.isNullOrEmpty(oldMetricsFileSuffix)) { oldMetricsFileSuffix = metricsFileSuffix; } else { oldMetricsFileSuffix += "." + metricsFileSuffix; } metricsReportingProps.setProperty(ConfigurationKeys.METRICS_FILE_SUFFIX, oldMetricsFileSuffix); startMetricReporting(metricsReportingProps); } /** * Start metric reporting. * * @param configuration configuration properties */ public void startMetricReporting(Configuration configuration) { Properties props = new Properties(); for (Map.Entry<String, String> entry : configuration) { props.put(entry.getKey(), entry.getValue()); } startMetricReporting(props); } /** * Start metric reporting. * * @param properties configuration properties */ public void startMetricReporting(Properties properties) { if (this.metricsReportingStarted) { LOGGER.warn("Metric reporting has already started"); return; } TimeUnit reportTimeUnit = TimeUnit.MILLISECONDS; long reportInterval = Long.parseLong(properties.getProperty(ConfigurationKeys.METRICS_REPORT_INTERVAL_KEY, ConfigurationKeys.DEFAULT_METRICS_REPORT_INTERVAL)); ScheduledReporter.setReportingInterval(properties, reportInterval, reportTimeUnit); long startTime = System.currentTimeMillis(); try { // Build and start the JMX reporter buildJmxMetricReporter(properties); if (this.jmxReporter.isPresent()) { LOGGER.info("Will start reporting metrics to JMX"); this.jmxReporter.get().start(); } // Build all other reporters buildFileMetricReporter(properties); buildKafkaMetricReporter(properties); buildGraphiteMetricReporter(properties); buildInfluxDBMetricReporter(properties); buildCustomMetricReporters(properties); buildFileFailureEventReporter(properties); // Start reporters that implement org.apache.gobblin.metrics.report.ScheduledReporter RootMetricContext.get().startReporting(); // Start reporters that implement com.codahale.metrics.ScheduledReporter for (com.codahale.metrics.ScheduledReporter scheduledReporter : this.codahaleScheduledReporters) { scheduledReporter.start(reportInterval, reportTimeUnit); } } catch (Exception e) { LOGGER.error("Metrics reporting cannot be started due to {}", ExceptionUtils.getFullStackTrace(e)); throw e; } this.metricsReportingStarted = true; LOGGER.info("Metrics reporting has been started in {} ms: GobblinMetrics {}", System.currentTimeMillis() - startTime, this.toString()); } /** * Stop metric reporting. */ public void stopMetricsReporting() { LOGGER.info("Metrics reporting will be stopped: GobblinMetrics {}", this.toString()); if (!this.metricsReportingStarted) { LOGGER.warn("Metric reporting has not started yet"); return; } // Stop the JMX reporter if (this.jmxReporter.isPresent()) { this.jmxReporter.get().stop(); } // Trigger and stop reporters that implement org.apache.gobblin.metrics.report.ScheduledReporter RootMetricContext.get().stopReporting(); // Trigger and stop reporters that implement com.codahale.metrics.ScheduledReporter for (com.codahale.metrics.ScheduledReporter scheduledReporter : this.codahaleScheduledReporters) { scheduledReporter.report(); } try { this.codahaleReportersCloser.close(); } catch (IOException ioe) { LOGGER.error("Failed to close metric output stream for job " + this.id, ioe); } catch (Exception e) { LOGGER.error("Failed to close metric output stream for job {} due to {}", this.id, ExceptionUtils.getFullStackTrace(e)); throw e; } this.metricsReportingStarted = false; // Remove from the cache registry GobblinMetrics.remove(id); LOGGER.info("Metrics reporting stopped successfully"); } private void buildFileMetricReporter(Properties properties) { if (!Boolean.valueOf(properties.getProperty(ConfigurationKeys.METRICS_REPORTING_FILE_ENABLED_KEY, ConfigurationKeys.DEFAULT_METRICS_REPORTING_FILE_ENABLED))) { return; } LOGGER.info("Reporting metrics to log files"); if (!properties.containsKey(ConfigurationKeys.METRICS_LOG_DIR_KEY)) { LOGGER.error("Not reporting metrics to log files because " + ConfigurationKeys.METRICS_LOG_DIR_KEY + " is undefined"); return; } try { String fsUri = properties.getProperty(ConfigurationKeys.FS_URI_KEY, ConfigurationKeys.LOCAL_FS_URI); FileSystem fs = FileSystem.get(URI.create(fsUri), new Configuration()); // Each job gets its own metric log subdirectory Path metricsLogDir = new Path(properties.getProperty(ConfigurationKeys.METRICS_LOG_DIR_KEY), this.getName()); if (!fs.exists(metricsLogDir) && !fs.mkdirs(metricsLogDir)) { LOGGER.error("Failed to create metric log directory for metrics " + this.getName()); return; } // Add a suffix to file name if specified in properties. String metricsFileSuffix = properties.getProperty(ConfigurationKeys.METRICS_FILE_SUFFIX, ConfigurationKeys.DEFAULT_METRICS_FILE_SUFFIX); if (!Strings.isNullOrEmpty(metricsFileSuffix) && !metricsFileSuffix.startsWith(".")) { metricsFileSuffix = "." + metricsFileSuffix; } // Each job run gets its own metric log file Path metricLogFile = new Path(metricsLogDir, this.id + metricsFileSuffix + ".metrics.log"); boolean append = false; // Append to the metric file if it already exists if (fs.exists(metricLogFile)) { LOGGER.info(String.format("Metric log file %s already exists, appending to it", metricLogFile)); append = true; } OutputStream output = append ? fs.append(metricLogFile) : fs.create(metricLogFile, true); // Add metrics reporter OutputStreamReporter.Factory.newBuilder().outputTo(output).build(properties); // Set up events reporter at the same time!! this.codahaleScheduledReporters.add(this.codahaleReportersCloser.register( OutputStreamEventReporter.forContext(RootMetricContext.get()).outputTo(output).build())); LOGGER.info("Will start reporting metrics to directory " + metricsLogDir); } catch (IOException ioe) { LOGGER.error("Failed to build file metric reporter for job " + this.id, ioe); } } private void buildFileFailureEventReporter(Properties properties) { if (!properties.containsKey(ConfigurationKeys.FAILURE_LOG_DIR_KEY)) { LOGGER.error("Not reporting failure to log files because " + ConfigurationKeys.FAILURE_LOG_DIR_KEY + " is undefined"); return; } try { String fsUri = properties.getProperty(ConfigurationKeys.FS_URI_KEY, ConfigurationKeys.LOCAL_FS_URI); FileSystem fs = FileSystem.get(URI.create(fsUri), new Configuration()); // Each job gets its own log subdirectory Path failureLogDir = new Path(properties.getProperty(ConfigurationKeys.FAILURE_LOG_DIR_KEY), this.getName()); if (!fs.exists(failureLogDir) && !fs.mkdirs(failureLogDir)) { LOGGER.error("Failed to create failure log directory for metrics " + this.getName()); return; } // Add a suffix to file name if specified in properties. String metricsFileSuffix = properties.getProperty(ConfigurationKeys.METRICS_FILE_SUFFIX, ConfigurationKeys.DEFAULT_METRICS_FILE_SUFFIX); if (!Strings.isNullOrEmpty(metricsFileSuffix) && !metricsFileSuffix.startsWith(".")) { metricsFileSuffix = "." + metricsFileSuffix; } // Each job run gets its own failure log file Path failureLogFile = new Path(failureLogDir, this.id + metricsFileSuffix + ".failure.log"); this.codahaleScheduledReporters.add(this.codahaleReportersCloser .register(new FileFailureEventReporter(RootMetricContext.get(), fs, failureLogFile))); LOGGER.info("Will start reporting failure to directory " + failureLogDir); } catch (IOException ioe) { LOGGER.error("Failed to build file failure event reporter for job " + this.id, ioe); } } private void buildJmxMetricReporter(Properties properties) { if (!Boolean.valueOf(properties.getProperty(ConfigurationKeys.METRICS_REPORTING_JMX_ENABLED_KEY, ConfigurationKeys.DEFAULT_METRICS_REPORTING_JMX_ENABLED))) { return; } LOGGER.info("Reporting metrics to JMX"); this.jmxReporter = Optional .of(codahaleReportersCloser.register(JmxReporter.forRegistry(RootMetricContext.get()) .convertRatesTo(TimeUnit.SECONDS).convertDurationsTo(TimeUnit.MILLISECONDS).build())); } private void buildKafkaMetricReporter(Properties properties) { if (!Boolean.valueOf(properties.getProperty(ConfigurationKeys.METRICS_REPORTING_KAFKA_ENABLED_KEY, ConfigurationKeys.DEFAULT_METRICS_REPORTING_KAFKA_ENABLED))) { return; } buildScheduledReporter(properties, ConfigurationKeys.DEFAULT_METRICS_REPORTING_KAFKA_REPORTER_CLASS, Optional.of("Kafka")); } private void buildGraphiteMetricReporter(Properties properties) { boolean metricsEnabled = PropertiesUtils.getPropAsBoolean(properties, ConfigurationKeys.METRICS_REPORTING_GRAPHITE_METRICS_ENABLED_KEY, ConfigurationKeys.DEFAULT_METRICS_REPORTING_GRAPHITE_METRICS_ENABLED); if (metricsEnabled) { LOGGER.info("Reporting metrics to Graphite"); } boolean eventsEnabled = PropertiesUtils.getPropAsBoolean(properties, ConfigurationKeys.METRICS_REPORTING_GRAPHITE_EVENTS_ENABLED_KEY, ConfigurationKeys.DEFAULT_METRICS_REPORTING_GRAPHITE_EVENTS_ENABLED); if (eventsEnabled) { LOGGER.info("Reporting events to Graphite"); } if (!metricsEnabled && !eventsEnabled) { return; } try { Preconditions.checkArgument( properties.containsKey(ConfigurationKeys.METRICS_REPORTING_GRAPHITE_HOSTNAME), "Graphite hostname is missing."); } catch (IllegalArgumentException exception) { LOGGER.error("Not reporting to Graphite due to missing Graphite configuration(s).", exception); return; } String hostname = properties.getProperty(ConfigurationKeys.METRICS_REPORTING_GRAPHITE_HOSTNAME); int port = Integer.parseInt(properties.getProperty(ConfigurationKeys.METRICS_REPORTING_GRAPHITE_PORT, ConfigurationKeys.DEFAULT_METRICS_REPORTING_GRAPHITE_PORT)); GraphiteConnectionType connectionType; String type = properties.getProperty(ConfigurationKeys.METRICS_REPORTING_GRAPHITE_SENDING_TYPE, ConfigurationKeys.DEFAULT_METRICS_REPORTING_GRAPHITE_SENDING_TYPE).toUpperCase(); String prefix = properties.getProperty(ConfigurationKeys.METRICS_REPORTING_GRAPHITE_PREFIX, ConfigurationKeys.DEFAULT_METRICS_REPORTING_GRAPHITE_PREFIX); try { connectionType = GraphiteConnectionType.valueOf(type); } catch (IllegalArgumentException exception) { LOGGER.warn("Graphite Reporter connection type " + type + " not recognized. Will use TCP for sending.", exception); connectionType = GraphiteConnectionType.TCP; } if (metricsEnabled) { try { GraphiteReporter.Factory.newBuilder().withConnectionType(connectionType) .withConnection(hostname, port).withMetricContextName(this.metricContext.getName()) //contains the current job id .withMetricsPrefix(prefix).build(properties); } catch (IOException e) { LOGGER.error("Failed to create Graphite metrics reporter. Will not report metrics to Graphite.", e); } } if (eventsEnabled) { boolean emitValueAsKey = PropertiesUtils.getPropAsBoolean(properties, ConfigurationKeys.METRICS_REPORTING_GRAPHITE_EVENTS_VALUE_AS_KEY, ConfigurationKeys.DEFAULT_METRICS_REPORTING_GRAPHITE_EVENTS_VALUE_AS_KEY); String eventsPortProp = properties .getProperty(ConfigurationKeys.METRICS_REPORTING_GRAPHITE_EVENTS_PORT); int eventsPort = (eventsPortProp == null) ? (metricsEnabled ? port : Integer.parseInt(ConfigurationKeys.METRICS_REPORTING_GRAPHITE_PORT)) : Integer.parseInt(eventsPortProp); try { GraphiteEventReporter eventReporter = GraphiteEventReporter.Factory .forContext(RootMetricContext.get()).withConnectionType(connectionType) .withConnection(hostname, eventsPort).withPrefix(prefix).withEmitValueAsKey(emitValueAsKey) .build(); this.codahaleScheduledReporters.add(this.codahaleReportersCloser.register(eventReporter)); } catch (IOException e) { LOGGER.error("Failed to create Graphite event reporter. Will not report events to Graphite.", e); } } } private void buildInfluxDBMetricReporter(Properties properties) { boolean metricsEnabled = PropertiesUtils.getPropAsBoolean(properties, ConfigurationKeys.METRICS_REPORTING_INFLUXDB_METRICS_ENABLED_KEY, ConfigurationKeys.DEFAULT_METRICS_REPORTING_INFLUXDB_METRICS_ENABLED); if (metricsEnabled) { LOGGER.info("Reporting metrics to InfluxDB"); } boolean eventsEnabled = PropertiesUtils.getPropAsBoolean(properties, ConfigurationKeys.METRICS_REPORTING_INFLUXDB_EVENTS_ENABLED_KEY, ConfigurationKeys.DEFAULT_METRICS_REPORTING_INFLUXDB_EVENTS_ENABLED); if (eventsEnabled) { LOGGER.info("Reporting events to InfluxDB"); } if (!metricsEnabled && !eventsEnabled) { return; } try { Preconditions.checkArgument( properties.containsKey(ConfigurationKeys.METRICS_REPORTING_INFLUXDB_DATABASE), "InfluxDB database name is missing."); } catch (IllegalArgumentException exception) { LOGGER.error("Not reporting to InfluxDB due to missing InfluxDB configuration(s).", exception); return; } String url = properties.getProperty(ConfigurationKeys.METRICS_REPORTING_INFLUXDB_URL); String username = properties.getProperty(ConfigurationKeys.METRICS_REPORTING_INFLUXDB_USER); String password = PasswordManager.getInstance(properties) .readPassword(properties.getProperty(ConfigurationKeys.METRICS_REPORTING_INFLUXDB_PASSWORD)); String database = properties.getProperty(ConfigurationKeys.METRICS_REPORTING_INFLUXDB_DATABASE); InfluxDBConnectionType connectionType; String type = properties.getProperty(ConfigurationKeys.METRICS_REPORTING_INFLUXDB_SENDING_TYPE, ConfigurationKeys.DEFAULT_METRICS_REPORTING_INFLUXDB_SENDING_TYPE).toUpperCase(); try { connectionType = InfluxDBConnectionType.valueOf(type); } catch (IllegalArgumentException exception) { LOGGER.warn("InfluxDB Reporter connection type " + type + " not recognized. Will use TCP for sending.", exception); connectionType = InfluxDBConnectionType.TCP; } if (metricsEnabled) { try { InfluxDBReporter.Factory.newBuilder().withConnectionType(connectionType) .withConnection(url, username, password, database) .withMetricContextName(this.metricContext.getName()) // contains the current job id .build(properties); } catch (IOException e) { LOGGER.error("Failed to create InfluxDB metrics reporter. Will not report metrics to InfluxDB.", e); } } if (eventsEnabled) { String eventsDbProp = properties .getProperty(ConfigurationKeys.METRICS_REPORTING_INFLUXDB_EVENTS_DATABASE); String eventsDatabase = (eventsDbProp == null) ? (metricsEnabled ? database : null) : eventsDbProp; try { InfluxDBEventReporter eventReporter = InfluxDBEventReporter.Factory .forContext(RootMetricContext.get()).withConnectionType(connectionType) .withConnection(url, username, password, eventsDatabase).build(); this.codahaleScheduledReporters.add(this.codahaleReportersCloser.register(eventReporter)); } catch (IOException e) { LOGGER.error("Failed to create InfluxDB event reporter. Will not report events to InfluxDB.", e); } } } /** * Build scheduled metrics reporters by reflection from the property * {@link org.apache.gobblin.configuration.ConfigurationKeys#METRICS_CUSTOM_BUILDERS}. This allows users to specify custom * reporters for Gobblin runtime without having to modify the code. */ private void buildCustomMetricReporters(Properties properties) { String reporterClasses = properties.getProperty(ConfigurationKeys.METRICS_CUSTOM_BUILDERS); if (Strings.isNullOrEmpty(reporterClasses)) { return; } for (String reporterClass : Splitter.on(",").split(reporterClasses)) { buildScheduledReporter(properties, reporterClass, Optional.<String>absent()); } } private void buildScheduledReporter(Properties properties, String reporterClass, Optional<String> reporterSink) { try { Class<?> clazz = Class.forName(reporterClass); if (CustomCodahaleReporterFactory.class.isAssignableFrom(clazz)) { CustomCodahaleReporterFactory customCodahaleReporterFactory = ((CustomCodahaleReporterFactory) clazz .getConstructor().newInstance()); com.codahale.metrics.ScheduledReporter scheduledReporter = customCodahaleReporterFactory .newScheduledReporter(RootMetricContext.get(), properties); if (scheduledReporter == null) { LOGGER.warn("Factory {} returns a null scheduledReporter", clazz.getSimpleName()); return; } this.codahaleReportersCloser.register(scheduledReporter); String reporterSinkMsg = reporterSink.isPresent() ? "to " + reporterSink.get() : ""; LOGGER.info("Will start reporting metrics " + reporterSinkMsg + " using " + reporterClass); this.codahaleScheduledReporters.add(scheduledReporter); } else if (CustomReporterFactory.class.isAssignableFrom(clazz)) { CustomReporterFactory customReporterFactory = ((CustomReporterFactory) clazz.getConstructor() .newInstance()); customReporterFactory.newScheduledReporter(properties); LOGGER.info("Will start reporting metrics using " + reporterClass); } else { throw new IllegalArgumentException("Class " + reporterClass + " specified by key " + ConfigurationKeys.METRICS_CUSTOM_BUILDERS + " must implement: " + CustomCodahaleReporterFactory.class + " or " + CustomReporterFactory.class); } } catch (ClassNotFoundException exception) { LOGGER.warn( String.format("Failed to create metric reporter: requested CustomReporterFactory %s not found.", reporterClass), exception); } catch (NoSuchMethodException exception) { LOGGER.warn(String.format("Failed to create metric reporter: requested CustomReporterFactory %s " + "does not have parameterless constructor.", reporterClass), exception); } catch (Exception exception) { LOGGER.warn("Could not create metric reporter from builder " + reporterClass + ".", exception); } } }