com.vmware.photon.controller.model.adapters.azure.stats.AzureComputeStatsGatherer.java Source code

Java tutorial

Introduction

Here is the source code for com.vmware.photon.controller.model.adapters.azure.stats.AzureComputeStatsGatherer.java

Source

/*
 * Copyright (c) 2015-2016 VMware, Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License.  You may obtain a copy of
 * the License at http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed
 * under the License is distributed on an "AS IS" BASIS, without warranties or
 * conditions of any kind, EITHER EXPRESS OR IMPLIED.  See the License for the
 * specific language governing permissions and limitations under the License.
 */

package com.vmware.photon.controller.model.adapters.azure.stats;

import static com.vmware.photon.controller.model.adapters.azure.constants.AzureConstants.AZURE_DIAGNOSTIC_STORAGE_ACCOUNT_LINK;
import static com.vmware.photon.controller.model.adapters.azure.utils.AzureUtils.awaitTermination;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.InvalidKeyException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;

import com.microsoft.azure.credentials.ApplicationTokenCredentials;
import com.microsoft.azure.storage.CloudStorageAccount;
import com.microsoft.azure.storage.ResultSegment;
import com.microsoft.azure.storage.StorageException;
import com.microsoft.azure.storage.table.CloudTable;
import com.microsoft.azure.storage.table.CloudTableClient;
import com.microsoft.azure.storage.table.DynamicTableEntity;
import com.microsoft.azure.storage.table.EntityProperty;
import com.microsoft.azure.storage.table.TableQuery;
import com.microsoft.azure.storage.table.TableQuery.Operators;
import com.microsoft.azure.storage.table.TableQuery.QueryComparisons;

import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;

import com.vmware.photon.controller.model.adapterapi.ComputeStatsRequest;
import com.vmware.photon.controller.model.adapterapi.ComputeStatsResponse.ComputeStats;
import com.vmware.photon.controller.model.adapters.azure.AzureUriPaths;
import com.vmware.photon.controller.model.adapters.azure.constants.AzureConstants;
import com.vmware.photon.controller.model.adapters.azure.model.stats.AzureMetricRequest;
import com.vmware.photon.controller.model.adapters.azure.model.stats.AzureMetricResponse;
import com.vmware.photon.controller.model.adapters.azure.model.stats.Datapoint;
import com.vmware.photon.controller.model.adapters.azure.model.stats.Location;
import com.vmware.photon.controller.model.adapters.azure.model.stats.MetricAvailability;
import com.vmware.photon.controller.model.adapters.azure.model.stats.MetricDefinitions;
import com.vmware.photon.controller.model.adapters.azure.model.stats.TableInfo;
import com.vmware.photon.controller.model.adapters.azure.utils.AzureStatsNormalizer;
import com.vmware.photon.controller.model.adapters.azure.utils.AzureUtils;
import com.vmware.photon.controller.model.adapters.util.AdapterUtils;
import com.vmware.photon.controller.model.constants.PhotonModelConstants;
import com.vmware.photon.controller.model.resources.ComputeService.ComputeStateWithDescription;
import com.vmware.photon.controller.model.resources.DiskService.DiskState;
import com.vmware.photon.controller.model.resources.StorageDescriptionService.StorageDescription;
import com.vmware.photon.controller.model.tasks.monitoring.SingleResourceStatsCollectionTaskService.SingleResourceStatsCollectionTaskState;
import com.vmware.photon.controller.model.tasks.monitoring.SingleResourceStatsCollectionTaskService.SingleResourceTaskCollectionStage;

import com.vmware.xenon.common.Operation;
import com.vmware.xenon.common.OperationContext;
import com.vmware.xenon.common.ServiceHost.ServiceNotFoundException;
import com.vmware.xenon.common.ServiceStats.ServiceStat;
import com.vmware.xenon.common.StatelessService;
import com.vmware.xenon.common.UriUtils;
import com.vmware.xenon.common.Utils;
import com.vmware.xenon.services.common.AuthCredentialsService;
import com.vmware.xenon.services.common.AuthCredentialsService.AuthCredentialsServiceState;

/**
 * This service queries the Azure storage table account to obtain the metrics of a VM.
 * It is called per compute VM to obtain the metrics.
 *
 */
public class AzureComputeStatsGatherer extends StatelessService {
    public static final String SELF_LINK = AzureUriPaths.AZURE_COMPUTE_STATS_GATHERER;
    private static final String PARTITION_KEY = "PartitionKey";
    private static final String COUNTER_NAME_KEY = "CounterName";
    private static final String TIMESTAMP = "Timestamp";
    public static final String[] METRIC_NAMES = { AzureConstants.NETWORK_BYTES_IN, AzureConstants.NETWORK_BYTES_OUT,
            AzureConstants.DISK_WRITES_PER_SECOND, AzureConstants.DISK_READS_PER_SECOND,
            AzureConstants.CPU_UTILIZATION, AzureConstants.MEMORY_AVAILABLE, AzureConstants.MEMORY_USED,
            AzureConstants.PERCENT_MEMORY_AVAILABLE, AzureConstants.PERCENT_MEMORY_USED,
            AzureConstants.DISK_READ_BYTES_PER_SECOND, AzureConstants.DISK_WRITE_BYTES_PER_SECOND };

    private ExecutorService executorService;

    @Override
    public void handleStart(Operation startPost) {
        this.executorService = getHost().allocateExecutor(this);
        super.handleStart(startPost);
    }

    @Override
    public void handleStop(Operation delete) {
        this.executorService.shutdown();
        awaitTermination(this, this.executorService);
        super.handleStop(delete);
    }

    private class AzureStatsDataHolder {
        public ComputeStateWithDescription computeDesc;
        public ComputeStateWithDescription parentDesc;
        public DiskState bootDisk;
        public StorageDescription storageDescripton;
        public AuthCredentialsService.AuthCredentialsServiceState storageAccountAuth;
        public AuthCredentialsService.AuthCredentialsServiceState parentAuth;
        public ComputeStatsRequest statsRequest;
        public ApplicationTokenCredentials credentials;
        public ComputeStats statsResponse;
        public AtomicInteger numResponses = new AtomicInteger(0);
        public String tableName;
        public String partitionValue;

        public AzureStatsDataHolder() {
            this.statsResponse = new ComputeStats();
            // create a thread safe map to hold stats values for resource
            this.statsResponse.statValues = new ConcurrentSkipListMap<>();
        }
    }

    @Override
    public void handlePatch(Operation op) {
        if (!op.hasBody()) {
            op.fail(new IllegalArgumentException("body is required"));
            return;
        }
        op.complete();
        ComputeStatsRequest statsRequest = op.getBody(ComputeStatsRequest.class);

        if (statsRequest.isMockRequest) {
            // patch status to parent task
            AdapterUtils.sendPatchToProvisioningTask(this, statsRequest.taskReference);
            return;
        }

        AzureStatsDataHolder statsData = new AzureStatsDataHolder();
        statsData.statsRequest = statsRequest;
        // TODO: https://jira-hzn.eng.vmware.com/browse/VSYM-1336
        getVMDescription(statsData);
    }

    private void getVMDescription(AzureStatsDataHolder statsData) {
        Consumer<Operation> onSuccess = (op) -> {
            statsData.computeDesc = op.getBody(ComputeStateWithDescription.class);
            getParentVMDescription(statsData);
        };
        URI computeUri = UriUtils.extendUriWithQuery(statsData.statsRequest.resourceReference,
                UriUtils.URI_PARAM_ODATA_EXPAND, Boolean.TRUE.toString());
        AdapterUtils.getServiceState(this, computeUri, onSuccess, getFailureConsumer(statsData));
    }

    private void getParentVMDescription(AzureStatsDataHolder statsData) {
        Consumer<Operation> onSuccess = (op) -> {
            statsData.parentDesc = op.getBody(ComputeStateWithDescription.class);
            getParentAuth(statsData);
        };
        URI computeUri = UriUtils.extendUriWithQuery(UriUtils.buildUri(getHost(), statsData.computeDesc.parentLink),
                UriUtils.URI_PARAM_ODATA_EXPAND, Boolean.TRUE.toString());
        AdapterUtils.getServiceState(this, computeUri, onSuccess, getFailureConsumer(statsData));
    }

    private void getParentAuth(AzureStatsDataHolder statsData) {
        Consumer<Operation> onSuccess = (op) -> {
            statsData.parentAuth = op.getBody(AuthCredentialsService.AuthCredentialsServiceState.class);
            getBootDisk(statsData);
        };
        String authLink = statsData.parentDesc.description.authCredentialsLink;
        AdapterUtils.getServiceState(this, authLink, onSuccess, getFailureConsumer(statsData));
    }

    private void getBootDisk(AzureStatsDataHolder statsData) {
        Consumer<Operation> onSuccess = (op) -> {
            statsData.bootDisk = op.getBody(DiskState.class);
            getStorageAccount(statsData);
        };
        /*
         * VSYM-655 - https://jira-hzn.eng.vmware.com/browse/VSYM-655
         * Until Azure design is finalized, the first and only disk will always be the boot disk.
         */
        if (statsData.computeDesc.diskLinks == null || statsData.computeDesc.diskLinks.isEmpty()) {
            AdapterUtils.sendFailurePatchToProvisioningTask(this, statsData.statsRequest.taskReference,
                    new IllegalStateException("No disks found"));
        }
        AdapterUtils.getServiceState(this, statsData.computeDesc.diskLinks.get(0), onSuccess,
                getFailureConsumer(statsData));
    }

    private void getStorageAccount(AzureStatsDataHolder statsData) {
        Consumer<Operation> onSuccess = (op) -> {
            statsData.storageDescripton = op.getBody(StorageDescription.class);
            getStorageAccountAuth(statsData);
        };
        AdapterUtils.getServiceState(this,
                statsData.computeDesc.customProperties.get(AZURE_DIAGNOSTIC_STORAGE_ACCOUNT_LINK), onSuccess,
                ((throwable) -> {
                    if (throwable instanceof ServiceNotFoundException) {
                        logInfo("Skipping stats collection because storage account not found for [%s]",
                                statsData.computeDesc.name);
                        patchEmptyResponse(statsData);
                        return;
                    }
                    AdapterUtils.sendFailurePatchToProvisioningTask(this, statsData.statsRequest.taskReference,
                            throwable);
                }));
    }

    private void getStorageAccountAuth(AzureStatsDataHolder statsData) {
        Consumer<Operation> onSuccess = (op) -> {
            statsData.storageAccountAuth = op.getBody(AuthCredentialsServiceState.class);
            getStats(statsData);
        };
        AdapterUtils.getServiceState(this, statsData.storageDescripton.authCredentialsLink, onSuccess,
                getFailureConsumer(statsData));
    }

    private Consumer<Throwable> getFailureConsumer(AzureStatsDataHolder statsData) {
        return ((throwable) -> {
            if (throwable instanceof ServiceNotFoundException) {
                logWarning("Skipping stats collection because [%s]", throwable.getMessage());
                patchEmptyResponse(statsData);
                return;
            }
            AdapterUtils.sendFailurePatchToProvisioningTask(this, statsData.statsRequest.taskReference, throwable);
        });
    }

    private void getAzureApplicationTokenCredential(AzureStatsDataHolder statsData) {
        if (statsData.credentials == null) {
            try {
                statsData.credentials = AzureUtils.getAzureConfig(statsData.parentAuth);
            } catch (Exception e) {
                logSevere(e);
            }
        }
    }

    private void getStats(AzureStatsDataHolder statsData) {
        getAzureApplicationTokenCredential(statsData);
        try {
            getMetricDefinitions(statsData);
        } catch (Exception e) {
            AdapterUtils.sendFailurePatchToProvisioningTask(this, statsData.statsRequest.taskReference, e);
        }
    }

    /**
     * Get the metric definitions from Azure using the Endpoint "/metricDefinitions"
     * The request and response of the API is as described in
     * {@link https://msdn.microsoft.com/en-us/library/azure/dn931939.aspx} Insights REST.
     * @param statsData
     * @throws URISyntaxException
     * @throws IOException
     */
    private void getMetricDefinitions(AzureStatsDataHolder statsData) throws URISyntaxException, IOException {
        String azureInstanceId = statsData.computeDesc.id;
        URI uri = UriUtils.buildUri(new URI(AzureConstants.BASE_URI_FOR_REST), azureInstanceId,
                AzureConstants.METRIC_DEFINITIONS_ENDPOINT);
        // Adding a filter to avoid huge data flow on the network
        /*
         * VSYM-656: https://jira-hzn.eng.vmware.com/browse/VSYM-656
         * Remove the filter when Unit of a metric is required.
         */
        uri = UriUtils.extendUriWithQuery(uri, AzureConstants.QUERY_PARAM_API_VERSION,
                AzureConstants.DIAGNOSTIC_SETTING_API_VERSION, AzureConstants.QUERY_PARAM_FILTER,
                AzureConstants.METRIC_DEFINITIONS_MEMORY_FILTER);
        Operation operation = Operation.createGet(uri);
        operation.addRequestHeader(Operation.ACCEPT_HEADER, Operation.MEDIA_TYPE_APPLICATION_JSON);
        operation.addRequestHeader(Operation.AUTHORIZATION_HEADER,
                AzureConstants.AUTH_HEADER_BEARER_PREFIX + statsData.credentials.getToken());
        operation.setCompletion((op, ex) -> {
            if (ex != null) {
                AdapterUtils.sendFailurePatchToProvisioningTask(this, statsData.statsRequest.taskReference, ex);
            }
            MetricDefinitions metricDefinitions = op.getBody(MetricDefinitions.class);
            DateTimeFormatter dateTimeFormatter = DateTimeFormat.forPattern(AzureConstants.METRIC_TIME_FORMAT);
            if (metricDefinitions.getValues() != null && !metricDefinitions.getValues().isEmpty()) {
                for (MetricAvailability metricAvailability : metricDefinitions.getValues().get(0)
                        .getMetricAvailabilities()) {
                    if (metricAvailability.getTimeGrain().equals(AzureConstants.METRIC_TIME_GRAIN_1_MINUTE)) {
                        Location location = metricAvailability.getLocation();
                        Date mostRecentTableDate = null;
                        for (TableInfo tableInfo : location.getTableInfo()) {
                            Date startDate = dateTimeFormatter.parseDateTime(tableInfo.getStartTime()).toDate();
                            if (mostRecentTableDate == null || startDate.after(mostRecentTableDate)) {
                                mostRecentTableDate = startDate;
                                statsData.tableName = tableInfo.getTableName();
                            }
                        }
                        statsData.partitionValue = location.getPartitionKey();
                    }
                }
            }
            if (statsData.tableName != null && statsData.tableName.length() > 0) {
                try {
                    getMetrics(statsData);
                } catch (Exception e) {
                    AdapterUtils.sendFailurePatchToProvisioningTask(this, statsData.statsRequest.taskReference, e);
                }
            } else {
                patchEmptyResponse(statsData);
            }
        });
        sendRequest(operation);
    }

    private void patchEmptyResponse(AzureStatsDataHolder statsData) {
        // Patch back to the Parent with empty response
        SingleResourceStatsCollectionTaskState respBody = new SingleResourceStatsCollectionTaskState();
        statsData.statsResponse.computeLink = statsData.computeDesc.documentSelfLink;
        respBody.taskStage = SingleResourceTaskCollectionStage.valueOf(statsData.statsRequest.nextStage);
        respBody.statsAdapterReference = UriUtils.buildUri(getHost(), SELF_LINK);
        respBody.statsList = new ArrayList<>();
        this.sendRequest(Operation.createPatch(statsData.statsRequest.taskReference).setBody(respBody));
    }

    private void getMetrics(AzureStatsDataHolder statsData)
            throws InvalidKeyException, URISyntaxException, StorageException {
        String storageAccountName = statsData.storageDescripton.name;
        String storageKey = statsData.storageAccountAuth.customProperties
                .get(AzureConstants.AZURE_STORAGE_ACCOUNT_KEY1);
        String storageConnectionString = String.format(AzureConstants.STORAGE_CONNECTION_STRING, storageAccountName,
                storageKey);
        for (String metricName : METRIC_NAMES) {
            AzureMetricRequest request = new AzureMetricRequest();
            request.setStorageConnectionString(storageConnectionString);
            request.setTableName(statsData.tableName);
            request.setPartitionValue(statsData.partitionValue);
            long endTimeMicros = Utils.getNowMicrosUtc();
            Date timeStamp = new Date(TimeUnit.MICROSECONDS.toMillis(endTimeMicros)
                    - TimeUnit.MINUTES.toMillis(AzureConstants.METRIC_COLLECTION_PERIOD));
            request.setTimestamp(timeStamp);
            request.setMetricName(metricName);
            AzureMetricsHandler handler = new AzureMetricsHandler(this, statsData);
            getMetricStatisticsAsync(request, handler);
        }
    }

    private class AzureMetricsHandler implements AsyncHandler<AzureMetricRequest, AzureMetricResponse> {
        private AzureStatsDataHolder statsData;
        private StatelessService service;
        private OperationContext opContext;

        public AzureMetricsHandler(StatelessService service, AzureStatsDataHolder statsData) {
            this.statsData = statsData;
            this.service = service;
            this.opContext = OperationContext.getOperationContext();
        }

        @Override
        public void onError(Exception exception) {
            OperationContext.restoreOperationContext(this.opContext);
            AdapterUtils.sendFailurePatchToProvisioningTask(this.service, this.statsData.statsRequest.taskReference,
                    exception);
        }

        @Override
        public void onSuccess(AzureMetricRequest request, AzureMetricResponse result) {
            OperationContext.restoreOperationContext(this.opContext);
            List<ServiceStat> statDatapoints = new ArrayList<>();
            List<Datapoint> dpList = result.getDatapoints();
            String normalizedMetricName = AzureStatsNormalizer.getNormalizedStatKeyValue(result.getLabel());
            if (dpList != null && dpList.size() != 0) {
                for (Datapoint dp : dpList) {
                    // TODO: https://jira-hzn.eng.vmware.com/browse/VSYM-769
                    ServiceStat stat = new ServiceStat();
                    stat.latestValue = dp.getAverage();
                    stat.sourceTimeMicrosUtc = TimeUnit.MILLISECONDS.toMicros(dp.getTimestamp().getTime());
                    stat.unit = PhotonModelConstants.getUnitForMetric(normalizedMetricName);
                    statDatapoints.add(stat);
                }

                this.statsData.statsResponse.statValues.put(normalizedMetricName, statDatapoints);
            }

            if (this.statsData.numResponses.incrementAndGet() == METRIC_NAMES.length) {
                SingleResourceStatsCollectionTaskState respBody = new SingleResourceStatsCollectionTaskState();
                this.statsData.statsResponse.computeLink = this.statsData.computeDesc.documentSelfLink;
                respBody.taskStage = SingleResourceTaskCollectionStage
                        .valueOf(this.statsData.statsRequest.nextStage);
                respBody.statsList = new ArrayList<>();
                respBody.statsList.add(this.statsData.statsResponse);
                respBody.statsAdapterReference = UriUtils.buildUri(getHost(), SELF_LINK);
                this.service.sendRequest(
                        Operation.createPatch(this.statsData.statsRequest.taskReference).setBody(respBody));
            }
        }
    }

    /**
     * Uses the executorService to kick of a new Callable,
     * which in turn patches back to the AsyncHandler that is passed.
     *
     * @param request
     * The request object
     * @param asyncHandler
     * The Asynchronous handler that will be called.
     * @return
     */
    public void getMetricStatisticsAsync(final AzureMetricRequest request,
            final AsyncHandler<AzureMetricRequest, AzureMetricResponse> asyncHandler) {
        this.executorService.submit(new Runnable() {
            @Override
            public void run() {
                AzureMetricResponse response = new AzureMetricResponse();
                try {
                    // Create the table client required to make calls to the table
                    CloudStorageAccount cloudStorageAccount = CloudStorageAccount
                            .parse(request.getStorageConnectionString());
                    CloudTableClient tableClient = cloudStorageAccount.createCloudTableClient();

                    // Get the table reference using the table name
                    CloudTable table = tableClient.getTableReference(request.getTableName());

                    // Create filters to limit the data
                    String partitionFilter = TableQuery.generateFilterCondition(PARTITION_KEY,
                            QueryComparisons.EQUAL, request.getPartitionValue());

                    String timestampFilter = TableQuery.generateFilterCondition(TIMESTAMP,
                            QueryComparisons.GREATER_THAN_OR_EQUAL, request.getTimestamp());

                    String partitionAndTimestampFilter = TableQuery.combineFilters(partitionFilter, Operators.AND,
                            timestampFilter);

                    String counterFilter = TableQuery.generateFilterCondition(COUNTER_NAME_KEY,
                            QueryComparisons.EQUAL, request.getMetricName());

                    // Combine all the filters
                    String combinedFilter = TableQuery.combineFilters(partitionAndTimestampFilter, Operators.AND,
                            counterFilter);

                    // Create the query
                    TableQuery<DynamicTableEntity> query = TableQuery.from(DynamicTableEntity.class)
                            .where(combinedFilter);

                    response.setLabel(request.getMetricName());
                    List<Datapoint> datapoints = new ArrayList<>();
                    ResultSegment<DynamicTableEntity> entities = table.executeSegmented(query, null);
                    for (DynamicTableEntity entity : entities.getResults()) {
                        HashMap<String, EntityProperty> properties = entity.getProperties();
                        Datapoint dp = new Datapoint();
                        for (String key : properties.keySet()) {
                            switch (key) {
                            case AzureConstants.METRIC_KEY_LAST:
                                dp.setLast(properties.get(key).getValueAsDoubleObject());
                                break;
                            case AzureConstants.METRIC_KEY_MAXIMUM:
                                dp.setMaximum(properties.get(key).getValueAsDoubleObject());
                                break;
                            case AzureConstants.METRIC_KEY_MINIMUM:
                                dp.setMinimum(properties.get(key).getValueAsDoubleObject());
                                break;
                            case AzureConstants.METRIC_KEY_COUNTER_NAME:
                                dp.setCounterName(properties.get(key).getValueAsString());
                                break;
                            case AzureConstants.METRIC_KEY_TIMESTAMP:
                                dp.setTimestamp(properties.get(key).getValueAsDate());
                                break;
                            case AzureConstants.METRIC_KEY_TOTAL:
                                dp.setTotal(properties.get(key).getValueAsDoubleObject());
                                break;
                            case AzureConstants.METRIC_KEY_AVERAGE:
                                dp.setAverage(properties.get(key).getValueAsDoubleObject());
                                break;
                            case AzureConstants.METRIC_KEY_COUNT:
                                dp.setCount(properties.get(key).getValueAsIntegerObject());
                                break;
                            default:
                                logInfo("Invalid key [%s]", key);
                                break;
                            }
                        }
                        datapoints.add(dp);
                        break;
                    }
                    response.setDatapoints(datapoints);
                } catch (Exception ex) {
                    if (asyncHandler != null) {
                        asyncHandler.onError(ex);
                    }
                }
                if (asyncHandler != null) {
                    asyncHandler.onSuccess(request, response);
                }
            }
        });
    }
}