io.pravega.service.server.host.stat.AutoScaleProcessor.java Source code

Java tutorial

Introduction

Here is the source code for io.pravega.service.server.host.stat.AutoScaleProcessor.java

Source

/**
 * Copyright (c) 2017 Dell Inc., or its subsidiaries. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 */
package io.pravega.service.server.host.stat;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.RemovalCause;
import com.google.common.cache.RemovalListener;
import com.google.common.cache.RemovalListeners;
import io.pravega.client.ClientFactory;
import io.pravega.client.segment.impl.Segment;
import io.pravega.client.stream.EventStreamWriter;
import io.pravega.client.stream.EventWriterConfig;
import io.pravega.client.stream.Serializer;
import io.pravega.client.stream.impl.JavaSerializer;
import io.pravega.common.util.Retry;
import io.pravega.shared.NameUtils;
import io.pravega.shared.controller.event.ScaleEvent;
import io.pravega.shared.protocol.netty.WireCommands;
import java.time.Duration;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;

/**
 * This looks at segment aggregates and determines if a scale operation has to be triggered.
 * If a scale has to be triggered, then it puts a new scale request into the request stream.
 */
@Slf4j
public class AutoScaleProcessor {

    private static final long TWO_MINUTES = Duration.ofMinutes(2).toMillis();
    private static final long FIVE_MINUTES = Duration.ofMinutes(5).toMillis();
    private static final long TEN_MINUTES = Duration.ofMinutes(10).toMillis();
    private static final long TWENTY_MINUTES = Duration.ofMinutes(20).toMillis();
    private static final int MAX_CACHE_SIZE = 1000000;
    private static final int INITIAL_CAPACITY = 1000;

    private final AtomicReference<ClientFactory> clientFactory = new AtomicReference<>();
    private final AtomicBoolean initialized = new AtomicBoolean(false);
    private final Cache<String, Pair<Long, Long>> cache;
    private final Serializer<ScaleEvent> serializer;
    private final AtomicReference<EventStreamWriter<ScaleEvent>> writer;
    private final EventWriterConfig writerConfig;
    private final AutoScalerConfig configuration;
    private final Executor executor;
    private final ScheduledExecutorService maintenanceExecutor;

    AutoScaleProcessor(AutoScalerConfig configuration, Executor executor,
            ScheduledExecutorService maintenanceExecutor) {
        this.configuration = configuration;
        this.maintenanceExecutor = maintenanceExecutor;
        this.executor = executor;

        serializer = new JavaSerializer<>();
        writerConfig = EventWriterConfig.builder().build();
        writer = new AtomicReference<>();

        cache = CacheBuilder.newBuilder().initialCapacity(INITIAL_CAPACITY).maximumSize(MAX_CACHE_SIZE)
                .expireAfterAccess(configuration.getCacheExpiry().getSeconds(), TimeUnit.SECONDS).removalListener(
                        RemovalListeners.asynchronous((RemovalListener<String, Pair<Long, Long>>) notification -> {
                            if (notification.getCause().equals(RemovalCause.EXPIRED)) {
                                triggerScaleDown(notification.getKey(), true);
                            }
                        }, maintenanceExecutor))
                .build();

        CompletableFuture.runAsync(this::bootstrapRequestWriters, maintenanceExecutor);
    }

    @VisibleForTesting
    AutoScaleProcessor(EventStreamWriter<ScaleEvent> writer, AutoScalerConfig configuration, Executor executor,
            ScheduledExecutorService maintenanceExecutor) {
        this(configuration, executor, maintenanceExecutor);
        this.writer.set(writer);
        this.initialized.set(true);
        maintenanceExecutor.scheduleAtFixedRate(cache::cleanUp, 0, configuration.getCacheCleanup().getSeconds(),
                TimeUnit.SECONDS);
    }

    @VisibleForTesting
    AutoScaleProcessor(AutoScalerConfig configuration, ClientFactory cf, Executor executor,
            ScheduledExecutorService maintenanceExecutor) {
        this(configuration, executor, maintenanceExecutor);
        clientFactory.set(cf);
    }

    private void bootstrapRequestWriters() {

        CompletableFuture<Void> createWriter = new CompletableFuture<>();

        // Starting with initial delay, in case request stream has not been created, to give it time to start
        // However, we have this wrapped in consumeFailure which means the creation of writer will be retried.
        // We are introducing a delay to avoid exceptions in the log in case creation of writer is attempted before
        // creation of requeststream.
        maintenanceExecutor
                .schedule(() -> Retry
                        .indefinitelyWithExpBackoff(100, 10, 10000,
                                e -> log.error("error while creating writer for requeststream {}", e))
                        .runAsync(() -> {
                            if (clientFactory.get() == null) {
                                clientFactory.compareAndSet(null, ClientFactory.withScope(
                                        NameUtils.INTERNAL_SCOPE_NAME, configuration.getControllerUri()));
                            }

                            this.writer.set(clientFactory.get().createEventWriter(
                                    configuration.getInternalRequestStream(), serializer, writerConfig));
                            initialized.set(true);
                            // even if there is no activity, keep cleaning up the cache so that scale down can be triggered.
                            // caches do not perform clean up if there is no activity. This is because they do not maintain their
                            // own background thread.
                            maintenanceExecutor.scheduleAtFixedRate(cache::cleanUp, 0,
                                    configuration.getCacheCleanup().getSeconds(), TimeUnit.SECONDS);
                            log.debug("bootstrapping auto-scale reporter done");
                            createWriter.complete(null);
                            return createWriter;
                        }, maintenanceExecutor), 10, TimeUnit.SECONDS);
    }

    private void triggerScaleUp(String streamSegmentName, int numOfSplits) {
        if (initialized.get()) {
            Pair<Long, Long> pair = cache.getIfPresent(streamSegmentName);
            long lastRequestTs = 0;

            if (pair != null && pair.getKey() != null) {
                lastRequestTs = pair.getKey();
            }

            long timestamp = System.currentTimeMillis();

            if (timestamp - lastRequestTs > configuration.getMuteDuration().toMillis()) {
                log.debug("sending request for scale up for {}", streamSegmentName);

                Segment segment = Segment.fromScopedName(streamSegmentName);
                ScaleEvent event = new ScaleEvent(segment.getScope(), segment.getStreamName(),
                        segment.getSegmentNumber(), ScaleEvent.UP, timestamp, numOfSplits, false);
                // Mute scale for timestamp for both scale up and down
                writeRequest(event)
                        .thenAccept(x -> cache.put(streamSegmentName, new ImmutablePair<>(timestamp, timestamp)));
            }
        }
    }

    private void triggerScaleDown(String streamSegmentName, boolean silent) {
        if (initialized.get()) {
            Pair<Long, Long> pair = cache.getIfPresent(streamSegmentName);
            long lastRequestTs = 0;

            if (pair != null && pair.getValue() != null) {
                lastRequestTs = pair.getValue();
            }

            long timestamp = System.currentTimeMillis();
            if (timestamp - lastRequestTs > configuration.getMuteDuration().toMillis()) {
                log.debug("sending request for scale down for {}", streamSegmentName);

                Segment segment = Segment.fromScopedName(streamSegmentName);
                ScaleEvent event = new ScaleEvent(segment.getScope(), segment.getStreamName(),
                        segment.getSegmentNumber(), ScaleEvent.DOWN, timestamp, 0, silent);
                writeRequest(event)
                        .thenAccept(x -> cache.put(streamSegmentName, new ImmutablePair<>(0L, timestamp)));
                // mute only scale downs
            }
        }
    }

    private CompletableFuture<Void> writeRequest(ScaleEvent event) {
        CompletableFuture<Void> result = new CompletableFuture<>();
        try {
            CompletableFuture.runAsync(() -> {
                try {
                    writer.get().writeEvent(event.getKey(), event).get();
                    result.complete(null);
                } catch (InterruptedException | ExecutionException e) {
                    log.error("error sending request to requeststream {}", e);
                    result.completeExceptionally(e);
                }
            }, executor);
        } catch (RejectedExecutionException e) {
            log.error("our executor queue is full. failed to post scale event for {}/{}/{}", event.getScope(),
                    event.getStream(), event.getSegmentNumber());
            result.completeExceptionally(e);
        }

        return result;
    }

    void report(String streamSegmentName, long targetRate, byte type, long startTime, double twoMinuteRate,
            double fiveMinuteRate, double tenMinuteRate, double twentyMinuteRate) {
        log.info("received traffic for {} with twoMinute rate = {} and targetRate = {}", streamSegmentName,
                twoMinuteRate, targetRate);
        if (initialized.get()) {
            // note: we are working on caller's thread. We should not do any blocking computation here and return as quickly as
            // possible.
            // So we will decide whether to scale or not and then unblock by asynchronously calling 'writeEvent'
            if (type != WireCommands.CreateSegment.NO_SCALE) {
                long currentTime = System.currentTimeMillis();
                if (currentTime - startTime > configuration.getCooldownDuration().toMillis()) {
                    log.debug("cool down period elapsed for {}", streamSegmentName);

                    // report to see if a scale operation needs to be performed.
                    if ((twoMinuteRate > 5.0 * targetRate && currentTime - startTime > TWO_MINUTES)
                            || (fiveMinuteRate > 2.0 * targetRate && currentTime - startTime > FIVE_MINUTES)
                            || (tenMinuteRate > targetRate && currentTime - startTime > TEN_MINUTES)) {
                        int numOfSplits = (int) (Double.max(Double.max(twoMinuteRate, fiveMinuteRate),
                                tenMinuteRate) / targetRate);
                        log.debug("triggering scale up for {}", streamSegmentName);

                        triggerScaleUp(streamSegmentName, numOfSplits);
                    }

                    if (twoMinuteRate < targetRate && fiveMinuteRate < targetRate && tenMinuteRate < targetRate
                            && twentyMinuteRate < targetRate / 2.0 && currentTime - startTime > TWENTY_MINUTES) {
                        log.debug("triggering scale down for {}", streamSegmentName);

                        triggerScaleDown(streamSegmentName, false);
                    }
                }
            }
        }
    }

    void notifyCreated(String segmentStreamName, byte type, long targetRate) {
        if (type != WireCommands.CreateSegment.NO_SCALE) {
            cache.put(segmentStreamName,
                    new ImmutablePair<>(System.currentTimeMillis(), System.currentTimeMillis()));
        }
    }

    void notifySealed(String segmentStreamName) {
        cache.invalidate(segmentStreamName);
    }

    @VisibleForTesting
    void put(String streamSegmentName, ImmutablePair<Long, Long> lrImmutablePair) {
        cache.put(streamSegmentName, lrImmutablePair);
    }

}