com.alibaba.jstorm.config.SupervisorRefreshConfig.java Source code

Introduction

Here is the source code for com.alibaba.jstorm.config.SupervisorRefreshConfig.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.alibaba.jstorm.config;

import backtype.storm.utils.NimbusClientWrapper;
import com.alibaba.jstorm.callback.RunnableCallback;
import com.alibaba.jstorm.client.ConfigExtension;
import com.alibaba.jstorm.utils.JStormUtils;
import com.alibaba.jstorm.utils.LoadConf;
import com.google.common.collect.MapDifference;
import com.google.common.collect.Maps;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * @author Cody (weiyue.wy@alibaba-inc.com)
 * @since 2.1.1
 */
public class SupervisorRefreshConfig extends RunnableCallback {
    private static final Logger LOG = LoggerFactory.getLogger(SupervisorRefreshConfig.class);

    /**
     * storm conf, excluding yarn config
     */
    private Map stormConf = new HashMap();

    private Map supervisorConf = new HashMap<>();

    /**
     * storm yaml string, excluding yarn config
     */
    private String stormYaml;

    private NimbusClientWrapper nimbusClientWrapper;
    private final Random random = new Random(System.currentTimeMillis());
    private boolean enableSync;
    private final Integer refreshInterval;
    private final YarnConfigBlacklist yarnConfigBlacklist;

    /**
     * yarn config
     */
    private final String retainedYarnConfig;

    public SupervisorRefreshConfig(Map conf) {
        LOG.info("init SupervisorRefreshConfig thread...");
        this.yarnConfigBlacklist = YarnConfigBlacklist.getInstance(conf);

        try {
            this.enableSync = ConfigExtension.getClusterConfSyncEnabled(conf);

            String rawYaml = FileUtils.readFileToString(new File(LoadConf.getStormYamlPath()));
            this.stormYaml = JStormUtils.trimEnd(yarnConfigBlacklist.filterConfigIfNecessary(rawYaml));
            this.stormConf.putAll(conf);
            this.stormConf.putAll(LoadConf.loadYamlFromString(this.stormYaml));

            this.supervisorConf.putAll(LoadConf.loadYamlFromString(this.stormYaml));
            this.retainedYarnConfig = yarnConfigBlacklist.getRetainedConfig(rawYaml);
            LOG.info("retained yarn config:\n============================\n{}", retainedYarnConfig);
        } catch (IOException ex) {
            LOG.error("failed to read local storm.yaml!", ex);
            throw new RuntimeException(ex);
        }

        // check nimbus config every 20 ~ 30 sec
        this.refreshInterval = random.nextInt(10) + 20;
        LOG.info("done.");
    }

    @Override
    public void run() {
        try {
            if (!enableSync) {
                return;
            }

            if (this.nimbusClientWrapper == null) {
                this.nimbusClientWrapper = new NimbusClientWrapper();
                try {
                    this.nimbusClientWrapper.init(this.stormConf);
                } catch (Exception ex) {
                    LOG.error("init nimbus client wrapper error, maybe nimbus is not alive.");
                }
            }

            String nimbusYaml = JStormUtils.trimEnd(yarnConfigBlacklist
                    .filterConfigIfNecessary(this.nimbusClientWrapper.getClient().getStormRawConf()));
            Map nimbusConf = LoadConf.loadYamlFromString(nimbusYaml);

            if (nimbusYaml != null && !this.supervisorConf.equals(nimbusConf)) {
                Map newConf = LoadConf.loadYamlFromString(nimbusYaml);
                if (newConf == null) {
                    LOG.error("received invalid storm.yaml, skip...");
                } else {
                    MapDifference<?, ?> diff = Maps.difference(this.supervisorConf, nimbusConf);
                    LOG.debug("conf diff, left only:{}, right only:{}", diff.entriesOnlyOnLeft(),
                            diff.entriesOnlyOnRight());
                    LOG.debug("received nimbus config update, new config:\n{}", nimbusYaml);
                    this.stormYaml = nimbusYaml;

                    // append yarn config
                    nimbusYaml = JStormUtils.trimEnd(nimbusYaml + "\n" + retainedYarnConfig);

                    // backup config & overwrite current storm.yaml
                    RefreshableComponents.refresh(newConf);
                    LoadConf.backupAndOverwriteStormYaml(nimbusYaml);
                }
            }
        } catch (Exception ex) {
            LOG.error("failed to get nimbus conf, maybe nimbus is not alive.");
            this.nimbusClientWrapper.reconnect();
        }
    }

    @Override
    public Object getResult() {
        return refreshInterval;
    }
}