org.hawkular.metrics.clients.ptrans.fullstack.CollectdITest.java Source code

Java tutorial

Introduction

Here is the source code for org.hawkular.metrics.clients.ptrans.fullstack.CollectdITest.java

Source

/*
 * Copyright 2014-2015 Red Hat, Inc. and/or its affiliates
 * and other contributors as indicated by the @author tags.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.hawkular.metrics.clients.ptrans.fullstack;

import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;
import static java.util.stream.Collectors.counting;
import static java.util.stream.Collectors.joining;
import static java.util.stream.Collectors.toList;

import static org.hawkular.metrics.clients.ptrans.ConfigurationKey.BATCH_DELAY;
import static org.hawkular.metrics.clients.ptrans.ConfigurationKey.BATCH_SIZE;
import static org.hawkular.metrics.clients.ptrans.ConfigurationKey.REST_URL;
import static org.hawkular.metrics.clients.ptrans.ConfigurationKey.SERVICES;
import static org.hawkular.metrics.clients.ptrans.util.ProcessUtil.kill;
import static org.hawkular.metrics.clients.ptrans.util.TenantUtil.getRandomTenantId;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.junit.Assume.assumeTrue;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Comparator;
import java.util.List;
import java.util.Locale;
import java.util.Properties;
import java.util.stream.Stream;

import org.hawkular.metrics.clients.ptrans.ExecutableITestBase;
import org.hawkular.metrics.clients.ptrans.PrintOutputOnFailureWatcher;
import org.hawkular.metrics.clients.ptrans.Service;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.type.CollectionType;
import com.fasterxml.jackson.databind.type.TypeFactory;
import com.google.common.collect.ImmutableList;
import com.google.common.io.Resources;

import jnr.constants.platform.Signal;

/**
 * @author Thomas Segismont
 */
public class CollectdITest extends ExecutableITestBase {
    private static final String COLLECTD_PATH = System.getProperty("collectd.path", "/usr/sbin/collectd");
    private static final String BASE_URI = System.getProperty("hawkular-metrics.base-uri",
            "127.0.0.1:8080/hawkular-metrics");

    private String tenant;
    private String findNumericMetricsUrl;
    private File collectdConfFile;
    private File collectdOut;
    private File collectdErr;
    private ProcessBuilder collectdProcessBuilder;
    private Process collectdProcess;

    @Rule
    public final PrintOutputOnFailureWatcher collectdOutputRule = new PrintOutputOnFailureWatcher("collectd",
            () -> collectdOut, () -> collectdErr);

    @Before
    public void setUp() throws Exception {
        tenant = getRandomTenantId();
        findNumericMetricsUrl = "http://" + BASE_URI + "/" + tenant + "/metrics?type=num";
        assumeCollectdIsPresent();
        configureCollectd();
        assertCollectdConfIsValid();
        configurePTrans();
    }

    private void assumeCollectdIsPresent() {
        Path path = Paths.get(COLLECTD_PATH);
        assumeTrue(COLLECTD_PATH + " does not exist", Files.exists(path));
        assumeTrue(COLLECTD_PATH + " is not a file", Files.isRegularFile(path));
        assumeTrue(COLLECTD_PATH + " is not executable", Files.isExecutable(path));
    }

    private void configureCollectd() throws Exception {
        collectdConfFile = temporaryFolder.newFile();
        try (FileOutputStream out = new FileOutputStream(collectdConfFile)) {
            Resources.copy(Resources.getResource("collectd.conf"), out);
        }
    }

    private void assertCollectdConfIsValid() throws Exception {
        collectdOut = temporaryFolder.newFile();
        collectdErr = temporaryFolder.newFile();

        collectdProcessBuilder = new ProcessBuilder();
        collectdProcessBuilder.directory(temporaryFolder.getRoot());
        collectdProcessBuilder.redirectOutput(collectdOut);
        collectdProcessBuilder.redirectError(collectdErr);

        collectdProcessBuilder.command(COLLECTD_PATH, "-C", collectdConfFile.getAbsolutePath(), "-t", "-T", "-f");

        collectdProcess = collectdProcessBuilder.start();

        int exitCode = collectdProcess.waitFor();
        assertEquals("Collectd configuration doesn't seem to be valid", 0, exitCode);

        boolean hasErrorInLog = Stream.concat(Files.lines(collectdOut.toPath()), Files.lines(collectdErr.toPath()))
                .anyMatch(l -> l.contains("[error]"));
        assertFalse("Collectd configuration doesn't seem to be valid", hasErrorInLog);
    }

    public void configurePTrans() throws Exception {
        Properties properties = new Properties();
        try (InputStream in = new FileInputStream(ptransConfFile)) {
            properties.load(in);
        }
        properties.setProperty(SERVICES.getExternalForm(), Service.COLLECTD.getExternalForm());
        properties.setProperty(BATCH_DELAY.getExternalForm(), String.valueOf(1));
        properties.setProperty(BATCH_SIZE.getExternalForm(), String.valueOf(1));
        String restUrl = "http://" + BASE_URI + "/" + tenant + "/metrics/numeric/data";
        properties.setProperty(REST_URL.getExternalForm(), restUrl);
        try (OutputStream out = new FileOutputStream(ptransConfFile)) {
            properties.store(out, "");
        }
    }

    @Test
    public void shouldFindCollectdMetricsOnServer() throws Exception {
        ptransProcessBuilder.command().addAll(ImmutableList.of("-c", ptransConfFile.getAbsolutePath()));
        ptransProcess = ptransProcessBuilder.start();
        assertPtransHasStarted(ptransProcess, ptransOut);

        File stdbuf = new File("/usr/bin/stdbuf");
        ImmutableList.Builder<String> collectdCmd = ImmutableList.builder();
        if (stdbuf.exists() && stdbuf.canExecute()) {
            collectdCmd.add(stdbuf.getAbsolutePath(), "-o0", "-e0");
        }
        collectdCmd.add(COLLECTD_PATH, "-C", collectdConfFile.getAbsolutePath(), "-f");
        collectdProcessBuilder.command(collectdCmd.build());
        collectdProcess = collectdProcessBuilder.start();

        waitForCollectdValues();

        kill(collectdProcess, Signal.SIGUSR1); // Flush data
        kill(collectdProcess, Signal.SIGTERM);
        collectdProcess.waitFor();

        Thread.sleep(MILLISECONDS.convert(1, SECONDS)); // Wait to make sure pTrans can send everything

        kill(ptransProcess, Signal.SIGTERM);
        ptransProcess.waitFor();

        List<Point> expectedData = getExpectedData();
        List<Point> serverData = getServerData();

        String failureMsg = String.format(Locale.ROOT, "Expected:%n%s%nActual:%n%s%n", pointsToString(expectedData),
                pointsToString(serverData));

        assertEquals(failureMsg, expectedData.size(), serverData.size());

        for (int i = 0; i < expectedData.size(); i++) {
            Point expectedPoint = expectedData.get(i);
            Point serverPoint = serverData.get(i);

            long timeDiff = expectedPoint.getTimestamp() - serverPoint.getTimestamp();
            assertTrue(failureMsg, Math.abs(timeDiff) < 2);

            assertEquals(failureMsg, expectedPoint.getType(), serverPoint.getType());
            assertEquals(failureMsg, expectedPoint.getValue(), serverPoint.getValue(), 0.1);
        }
    }

    private String pointsToString(List<Point> data) {
        return data.stream().map(Point::toString).collect(joining(System.getProperty("line.separator")));
    }

    private void waitForCollectdValues() throws Exception {
        long c;
        do {
            Thread.sleep(MILLISECONDS.convert(1, SECONDS));
            c = Files.lines(collectdOut.toPath()).filter(l -> l.startsWith("PUTVAL")).collect(counting());
        } while (c < 1);
    }

    private List<Point> getExpectedData() throws Exception {
        return Files.lines(collectdOut.toPath()).filter(l -> l.startsWith("PUTVAL")).map(this::collectdLogToPoint)
                .sorted(Comparator.comparing(Point::getType).thenComparing(Point::getTimestamp)).collect(toList());
    }

    private Point collectdLogToPoint(String line) {
        String[] split = line.split(" ");
        assertEquals("Unexpected format: " + line, 4, split.length);
        String[] metric = split[1].split("/");
        assertEquals("Unexpected format: " + line, 3, metric.length);
        metric = metric[2].split("-");
        assertEquals("Unexpected format: " + line, 2, metric.length);
        String[] data = split[3].split(":");
        assertEquals("Unexpected format: " + line, 2, data.length);
        long timestamp = Long.parseLong(data[0].replace(".", ""));
        if (!data[0].contains(".")) {
            // collectd v4 does not have high resolution timestamps
            timestamp *= 1000;
        }
        double value = Double.parseDouble(data[1]);
        return new Point(metric[1], timestamp, value);
    }

    private List<Point> getServerData() throws Exception {
        ObjectMapper objectMapper = new ObjectMapper();

        HttpURLConnection urlConnection = (HttpURLConnection) new URL(findNumericMetricsUrl).openConnection();
        urlConnection.connect();
        int responseCode = urlConnection.getResponseCode();
        if (responseCode != HttpURLConnection.HTTP_OK) {
            String msg = "Could not get metrics list from server: %s, %d";
            fail(String.format(Locale.ROOT, msg, findNumericMetricsUrl, responseCode));
        }
        List<String> metricNames;
        try (InputStream inputStream = urlConnection.getInputStream()) {
            TypeFactory typeFactory = objectMapper.getTypeFactory();
            CollectionType valueType = typeFactory.constructCollectionType(List.class, MetricName.class);
            List<MetricName> value = objectMapper.readValue(inputStream, valueType);
            metricNames = value.stream().map(MetricName::getId).collect(toList());
        }

        Stream<Point> points = Stream.empty();

        for (String metricName : metricNames) {
            String[] split = metricName.split("\\.");
            String type = split[split.length - 1];

            urlConnection = (HttpURLConnection) new URL(findNumericDataUrl(metricName)).openConnection();
            urlConnection.connect();
            responseCode = urlConnection.getResponseCode();
            if (responseCode != HttpURLConnection.HTTP_OK) {
                fail("Could not load metric data from server: " + responseCode);
            }

            try (InputStream inputStream = urlConnection.getInputStream()) {
                TypeFactory typeFactory = objectMapper.getTypeFactory();
                CollectionType valueType = typeFactory.constructCollectionType(List.class, MetricData.class);
                List<MetricData> data = objectMapper.readValue(inputStream, valueType);
                Stream<Point> metricPoints = data.stream()
                        .map(metricData -> new Point(type, metricData.timestamp, metricData.value));
                points = Stream.concat(points, metricPoints);
            }
        }

        return points.sorted(Comparator.comparing(Point::getType).thenComparing(Point::getTimestamp))
                .collect(toList());
    }

    private String findNumericDataUrl(String metricName) {
        return "http://" + BASE_URI + "/" + tenant + "/metrics/numeric/" + metricName + "/data";
    }

    @After
    public void tearDown() {
        if (collectdProcess != null && collectdProcess.isAlive()) {
            collectdProcess.destroy();
        }
    }

    private static final class Point {
        final String type;
        final long timestamp;
        final double value;

        Point(String type, long timestamp, double value) {
            this.type = type;
            this.timestamp = timestamp;
            this.value = value;
        }

        String getType() {
            return type;
        }

        long getTimestamp() {
            return timestamp;
        }

        double getValue() {
            return value;
        }

        @Override
        public String toString() {
            return "Point[" + "type='" + type + '\'' + ", timestamp=" + timestamp + ", value=" + value + ']';
        }
    }

    @SuppressWarnings("unused")
    private static final class MetricData {
        long timestamp;
        double value;

        public long getTimestamp() {
            return timestamp;
        }

        public void setTimestamp(long timestamp) {
            this.timestamp = timestamp;
        }

        public double getValue() {
            return value;
        }

        public void setValue(double value) {
            this.value = value;
        }
    }

    @JsonIgnoreProperties(ignoreUnknown = true)
    @SuppressWarnings("unused")
    private static final class MetricName {
        String id;

        public String getId() {
            return id;
        }

        public void setId(String id) {
            this.id = id;
        }
    }
}