io.hops.hopsworks.dela.DelaSetupWorker.java Source code

Java tutorial

Introduction

Here is the source code for io.hops.hopsworks.dela.DelaSetupWorker.java

Source

/*
 * Changes to this file committed after and not including commit-id: ccc0d2c5f9a5ac661e60e6eaf138de7889928b8b
 * are released under the following license:
 *
 * This file is part of Hopsworks
 * Copyright (C) 2018, Logical Clocks AB. All rights reserved
 *
 * Hopsworks is free software: you can redistribute it and/or modify it under the terms of
 * the GNU Affero General Public License as published by the Free Software Foundation,
 * either version 3 of the License, or (at your option) any later version.
 *
 * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
 * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
 * PURPOSE.  See the GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License along with this program.
 * If not, see <https://www.gnu.org/licenses/>.
 *
 * Changes to this file committed before and including commit-id: ccc0d2c5f9a5ac661e60e6eaf138de7889928b8b
 * are released under the following license:
 *
 * Copyright (C) 2013 - 2018, Logical Clocks AB and RISE SICS AB. All rights reserved
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of this
 * software and associated documentation files (the "Software"), to deal in the Software
 * without restriction, including without limitation the rights to use, copy, modify, merge,
 * publish, distribute, sublicense, and/or sell copies of the Software, and to permit
 * persons to whom the Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all copies or
 * substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS  OR IMPLIED, INCLUDING
 * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR  OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */
package io.hops.hopsworks.dela;

import com.google.gson.Gson;
import io.hops.hopsworks.common.dao.dela.certs.ClusterCertificateFacade;
import io.hops.hopsworks.common.dela.AddressJSON;
import io.hops.hopsworks.common.exception.RESTCodes;
import io.hops.hopsworks.common.security.CertificatesMgmService;
import io.hops.hopsworks.common.util.Settings;
import io.hops.hopsworks.dela.dto.hopssite.ClusterServiceDTO;
import io.hops.hopsworks.common.exception.DelaException;
import io.hops.hopsworks.dela.hopssite.HopssiteController;
import io.hops.hopsworks.util.CertificateHelper;
import io.hops.hopsworks.util.SettingsHelper;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.security.KeyStore;
import java.util.List;
import java.util.Optional;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import javax.annotation.Resource;
import javax.ejb.EJB;
import javax.ejb.Singleton;
import javax.ejb.Startup;
import javax.ejb.Timeout;
import javax.ejb.Timer;
import javax.ejb.TimerService;
import javax.ejb.TransactionAttribute;
import javax.ejb.TransactionAttributeType;
import org.apache.commons.codec.digest.DigestUtils;
import org.javatuples.Pair;
import org.javatuples.Triplet;

@Startup
@Singleton
public class DelaSetupWorker {

    private static final Logger LOGGER = Logger.getLogger(DelaSetupWorker.class.getName());

    @Resource
    TimerService timerService;
    @EJB
    private Settings settings;
    @EJB
    private ClusterCertificateFacade clusterCertFacade;
    @EJB
    private DelaStateController delaStateCtrl;
    @EJB
    private HopssiteController hopsSiteProxy;
    @EJB
    private TransferDelaController delaCtrl;
    @EJB
    private CertificatesMgmService certificatesMgmService;

    private State state;
    //5 required to get from start to running in perfect mode
    private int tryTimeouts = 0;
    private boolean healthy = true;
    /**
     * after 20 tries without reaching last state(active - running), backoff and try a bit less often, there is something
     * obviously wrong with some other third party service. .Give it time to regenerate
     */
    private static int TRY_TIMERS_BACKOFF = 20;

    @PostConstruct
    private void init() {
        if (delaStateCtrl.delaEnabled()) {
            state = State.SETUP;
            timerService.createTimer(0, settings.getHOPSSITE_HEARTBEAT_RETRY(), "Timer for dela settings check.");
            LOGGER.log(Level.INFO, "{0} - state:{1}", new Object[] { DelaException.Source.HOPS_SITE, state });
        }
    }

    @PreDestroy
    private void destroyTimer() {
        for (Timer timer : timerService.getTimers()) {
            timer.cancel();
        }
    }

    private Timer resetTimer(Timer timer, long intervalDuration) {
        timer.cancel();
        return timerService.createTimer(intervalDuration, intervalDuration, "Timer for " + state + ".");
    }

    @Timeout
    @TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED)
    private void timeout(Timer timer) {
        LOGGER.log(Level.INFO, "{0} - state:{1} timeout:{2}",
                new Object[] { DelaException.Source.HOPS_SITE, state, timer.getInfo().toString() });
        switch (state) {
        case SETUP:
            setup(timer);
            break;
        case DELA_VERSION:
            delaVersion(timer);
            break;
        case DELA_CONTACT:
            delaContact(timer);
            break;
        case REGISTER:
            hopsSiteRegister(timer);
            break;
        case HEAVY_PING:
            heavyPing(timer);
            break;
        case PING:
            ping(timer);
            break;
        default:
            throw new IllegalStateException("unknown state");
        }
    }

    //********************************************************************************************************************
    private void setup(Timer timer) {
        Optional<String> masterPswd = settings.getHopsSiteClusterPswd();
        if (!masterPswd.isPresent()) {
            //TODO Alex - use the registration pswd hash once the admin UI is ready
            String pswd = DigestUtils.sha256Hex(settings.getHopsSiteClusterPswdAux());
            settings.setHopsSiteClusterPswd(pswd);
            masterPswd = settings.getHopsSiteClusterPswd();
        }
        Optional<String> clusterName = settings.getHopsSiteClusterName();

        if (clusterName.isPresent()) {
            Optional<Triplet<KeyStore, KeyStore, String>> keystoreAux = CertificateHelper.loadKeystoreFromDB(
                    masterPswd.get(), clusterName.get(), clusterCertFacade, certificatesMgmService);
            if (keystoreAux.isPresent()) {
                setupComplete(keystoreAux.get(), timer);
                return;
            }
        }

        Optional<Triplet<KeyStore, KeyStore, String>> keystoreAux = CertificateHelper
                .loadKeystoreFromFile(masterPswd.get(), settings, clusterCertFacade, certificatesMgmService);
        if (keystoreAux.isPresent()) {
            setupComplete(keystoreAux.get(), timer);
        } else {
            LOGGER.log(Level.WARNING, "{0} - dela setup not ready - certificates not ready",
                    new Object[] { DelaException.Source.HOPS_SITE });
        }
    }

    private void setupComplete(Triplet<KeyStore, KeyStore, String> keystoreAux, Timer timer) {
        KeyStore keystore = keystoreAux.getValue0();
        KeyStore truststore = keystoreAux.getValue1();
        String certPswd = keystoreAux.getValue2();
        delaStateCtrl.hopssiteCertsAvailable(keystore, truststore, certPswd);
        timer = resetToDelaVersion(timer);
        delaVersion(timer);
    }

    private String delaVersion;

    private void delaVersion(Timer timer) {
        LOGGER.log(Level.INFO, "{0} - retrieving hops-site dela_version",
                new Object[] { DelaException.Source.HOPS_SITE });
        try {
            delaVersion = hopsSiteProxy.delaVersion();
            delaStateCtrl.hopssiteContacted();
            timer = resetToDelaContact(timer);
            delaContact(timer);
        } catch (DelaException tpe) {
            LOGGER.log(Level.WARNING, "{0} - source:<{1}>:{2}",
                    new Object[] { DelaException.Source.HOPS_SITE, tpe.getSource(), tpe.getMessage() });
            timer = tryAgainLater(timer);
        }
    }

    private void delaContact(Timer timer) {
        LOGGER.log(Level.INFO, "{0} - state:{1}", new Object[] { DelaException.Source.HOPS_SITE, state });
        AddressJSON delaTransferEndpoint;
        try {
            delaTransferEndpoint = SettingsHelper.delaTransferEndpoint(settings);
        } catch (DelaException tpe) {
            try {
                delaTransferEndpoint = getDelaTransferEndpoint(delaVersion);
                delaStateCtrl.transferDelaContacted();
                settings.setDELA_PUBLIC_ENDPOINT(delaTransferEndpoint);
                timer = resetToRegister(timer);
                hopsSiteRegister(timer, delaTransferEndpoint);
            } catch (DelaException tpe2) {
                LOGGER.log(Level.WARNING, "{0} - source:<{1}>:{2}",
                        new Object[] { DelaException.Source.HOPS_SITE, tpe2.getSource(), tpe2.getMessage() });
                timer = tryAgainLater(timer);
            }
        }

    }

    private void hopsSiteRegister(Timer timer) {
        LOGGER.log(Level.INFO, "{0} - state:{1}", new Object[] { DelaException.Source.HOPS_SITE, state });
        AddressJSON delaTransferEndpoint;
        try {
            delaTransferEndpoint = SettingsHelper.delaTransferEndpoint(settings);
        } catch (DelaException ex) {
            timer = resetToDelaContact(timer);
            return;
        }
        try {
            hopsSiteRegister(timer, delaTransferEndpoint);
        } catch (DelaException tpe) {
            LOGGER.log(Level.WARNING, "{0} - source:<{1}>:{2}",
                    new Object[] { DelaException.Source.HOPS_SITE, tpe.getSource(), tpe.getMessage() });
            timer = tryAgainLater(timer);
        }
    }

    private void hopsSiteRegister(Timer timer, AddressJSON delaTransferEndpoint) throws DelaException {
        String delaHttpEndpoint = SettingsHelper.delaHttpEndpoint(settings);
        hopsSiteRegister(timer, delaHttpEndpoint, delaTransferEndpoint);
    }

    private void hopsSiteRegister(Timer timer, String delaClusterAddress, AddressJSON delaTransferAddress)
            throws DelaException {
        String publicCId = hopsSiteProxy.registerCluster(delaClusterAddress,
                new Gson().toJson(delaTransferAddress));
        settings.setDELA_CLUSTER_ID(publicCId);
        timer = resetToHeavyPing(timer);
        heavyPing(timer);
    }

    private void heavyPing(Timer timer) {
        LOGGER.log(Level.INFO, "{0} - state:{1}", new Object[] { DelaException.Source.HOPS_SITE, state });
        Pair<List<String>, List<String>> datasets;
        try {
            datasets = delaCtrl.getContents();
        } catch (DelaException tpe) {
            LOGGER.log(Level.WARNING, "{0} - source:<{1}>:{2}",
                    new Object[] { DelaException.Source.HOPS_SITE, tpe.getSource(), tpe.getMessage() });
            if (DelaException.Source.SETTINGS.equals(tpe.getSource())) {
                timer = resetToDelaContact(timer);
            } else {
                timer = tryAgainLater(timer);
            }
            return;
        }
        try {
            hopsSiteProxy.heavyPing(datasets.getValue0(), datasets.getValue1());
            timer = resetToPing(timer);
            ping(timer);
        } catch (DelaException tpe) {
            LOGGER.log(Level.WARNING, "{0} - source:<{1}>:{2}",
                    new Object[] { DelaException.Source.HOPS_SITE, tpe.getSource(), tpe.getMessage() });
            if (RESTCodes.DelaErrorCode.CLUSTER_NOT_REGISTERED.getMessage().equals(tpe.getMessage())) {
                timer = resetToRegister(timer);
            } else if (DelaException.Source.SETTINGS.equals(tpe.getSource())) {
                timer = resetToRegister(timer);
            } else {
                timer = tryAgainLater(timer);
            }
        }
    }

    private void ping(Timer timer) {
        LOGGER.log(Level.INFO, "{0} - state:{1}", new Object[] { DelaException.Source.HOPS_SITE, state });
        Pair<List<String>, List<String>> datasets;
        try {
            datasets = delaCtrl.getContents();
        } catch (DelaException tpe) {
            LOGGER.log(Level.WARNING, "{0} - source:<{1}>:{2}",
                    new Object[] { DelaException.Source.HOPS_SITE, tpe.getSource(), tpe.getMessage() });
            if (DelaException.Source.SETTINGS.equals(tpe.getSource())) {
                timer = resetToDelaContact(timer);
            } else {
                timer = tryAgainLater(timer);
            }
            return;
        }
        try {
            hopsSiteProxy
                    .ping(new ClusterServiceDTO.Ping(datasets.getValue0().size(), datasets.getValue1().size()));
            if (!healthy) {
                timer = healthyPing(timer);
            }
        } catch (DelaException tpe) {
            LOGGER.log(Level.WARNING, "{0} - source:<{1}>:{2}",
                    new Object[] { DelaException.Source.HOPS_SITE, tpe.getSource(), tpe.getMessage() });
            if (RESTCodes.DelaErrorCode.CLUSTER_NOT_REGISTERED.getMessage().equals(tpe.getMessage())) {
                timer = resetToRegister(timer);
            } else if (RESTCodes.DelaErrorCode.HEAVY_PING.getMessage().equals(tpe.getMessage())) {
                timer = resetToHeavyPing(timer);
            } else if (DelaException.Source.SETTINGS.equals(tpe.getSource())) {
                timer = resetToDelaContact(timer);
            } else {
                timer = tryAgainLater(timer);
            }
        }
    }

    private Timer tryTimer(Timer timer, boolean health) {
        this.healthy = health;
        long intervalDuration;
        if (this.healthy) {
            tryTimeouts = 0;
            intervalDuration = settings.getHOPSSITE_HEARTBEAT_INTERVAL();
        } else {
            tryTimeouts++;
            if (tryTimeouts < TRY_TIMERS_BACKOFF) {
                intervalDuration = settings.getHOPSSITE_HEARTBEAT_RETRY();
            } else {
                //backoff and try again after a longer timeout
                intervalDuration = settings.getHOPSSITE_HEARTBEAT_INTERVAL();
            }
        }
        return resetTimer(timer, intervalDuration);
    }

    private Timer tryAgainLater(Timer timer) {
        return tryTimer(timer, false);
    }

    private Timer resetToSettings(Timer timer) {
        state = State.SETUP;
        return tryTimer(timer, false);
    }

    private Timer resetToDelaVersion(Timer timer) {
        LOGGER.log(Level.WARNING, "{0} - reset from:{1} to {2}",
                new Object[] { DelaException.Source.HOPS_SITE, state, State.DELA_VERSION });
        state = State.DELA_VERSION;
        return tryTimer(timer, false);
    }

    private Timer resetToDelaContact(Timer timer) {
        LOGGER.log(Level.WARNING, "{0} - reset from:{1} to {2}",
                new Object[] { DelaException.Source.HOPS_SITE, state, State.DELA_CONTACT });
        state = State.DELA_CONTACT;
        return tryTimer(timer, false);
    }

    private Timer resetToRegister(Timer timer) {
        LOGGER.log(Level.WARNING, "{0} - reset from:{1} to {2}",
                new Object[] { DelaException.Source.HOPS_SITE, state, State.REGISTER });
        state = State.REGISTER;
        return tryTimer(timer, false);
    }

    private Timer resetToHeavyPing(Timer timer) {
        LOGGER.log(Level.WARNING, "{0} - reset from:{1} to {2}",
                new Object[] { DelaException.Source.HOPS_SITE, state, State.HEAVY_PING });
        state = State.HEAVY_PING;
        return tryTimer(timer, false);
    }

    private Timer resetToPing(Timer timer) {
        LOGGER.log(Level.WARNING, "{0} - reset from:{1} to {2}",
                new Object[] { DelaException.Source.HOPS_SITE, state, State.PING });
        state = State.PING;
        return tryTimer(timer, false);
    }

    private Timer healthyPing(Timer timer) {
        state = State.PING;
        return tryTimer(timer, true);
    }

    private String getDelaTransferHttpEndpoint() throws HeartbeatException {
        String delaHttpEndpoint = settings.getDELA_TRANSFER_HTTP_ENDPOINT();
        try {
            URL u = new URL(delaHttpEndpoint);
            u.toURI();
            // validate url syntax
            if (u.getHost().isEmpty() || u.getPort() == -1) {
                LOGGER.log(Level.WARNING, "{0} - malformed dela url. {1} - reset in database",
                        new Object[] { DelaException.Source.HOPS_SITE, delaHttpEndpoint });
                throw new HeartbeatException("DELA_TRANSFER_HTTP");
            }
            return delaHttpEndpoint;
        } catch (MalformedURLException | URISyntaxException ex) {
            LOGGER.log(Level.SEVERE, "{1} - malformed dela url. {1} - reset in database",
                    new Object[] { DelaException.Source.HOPS_SITE, delaHttpEndpoint });
            throw new HeartbeatException("DELA_TRANSFER_HTTP");
        }
    }

    private AddressJSON getDelaTransferEndpoint(String delaVersion) throws DelaException {
        String delaTransferHttpEndpoint = SettingsHelper.delaTransferHttpEndpoint(settings);
        LOGGER.log(Level.INFO, "{0} - dela http endpoint: {1}",
                new Object[] { DelaException.Source.HOPS_SITE, delaTransferHttpEndpoint });
        AddressJSON delaTransferEndpoint = delaCtrl.getDelaPublicEndpoint(delaVersion);
        LOGGER.log(Level.INFO, "{0} - dela transfer endpoint: {1}",
                new Object[] { DelaException.Source.HOPS_SITE, delaTransferEndpoint.toString() });
        return delaTransferEndpoint;

    }

    private static class HeartbeatException extends Exception {

        public HeartbeatException(String msg) {
            super(msg);
        }
    }

    private static enum State {

        SETUP, DELA_VERSION, DELA_CONTACT, REGISTER, HEAVY_PING, PING
    }
}