io.hops.hopsworks.api.zeppelin.server.ZeppelinConfig.java Source code

Java tutorial

Introduction

Here is the source code for io.hops.hopsworks.api.zeppelin.server.ZeppelinConfig.java

Source

/*
 * Changes to this file committed after and not including commit-id: ccc0d2c5f9a5ac661e60e6eaf138de7889928b8b
 * are released under the following license:
 *
 * This file is part of Hopsworks
 * Copyright (C) 2018, Logical Clocks AB. All rights reserved
 *
 * Hopsworks is free software: you can redistribute it and/or modify it under the terms of
 * the GNU Affero General Public License as published by the Free Software Foundation,
 * either version 3 of the License, or (at your option) any later version.
 *
 * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
 * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
 * PURPOSE.  See the GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License along with this program.
 * If not, see <https://www.gnu.org/licenses/>.
 *
 * Changes to this file committed before and including commit-id: ccc0d2c5f9a5ac661e60e6eaf138de7889928b8b
 * are released under the following license:
 *
 * Copyright (C) 2013 - 2018, Logical Clocks AB and RISE SICS AB. All rights reserved
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of this
 * software and associated documentation files (the "Software"), to deal in the Software
 * without restriction, including without limitation the rights to use, copy, modify, merge,
 * publish, distribute, sublicense, and/or sell copies of the Software, and to permit
 * persons to whom the Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all copies or
 * substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS  OR IMPLIED, INCLUDING
 * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR  OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

package io.hops.hopsworks.api.zeppelin.server;

import com.github.eirslett.maven.plugins.frontend.lib.TaskRunnerException;
import io.hops.hopsworks.api.zeppelin.socket.NotebookServerImpl;
import io.hops.hopsworks.api.zeppelin.socket.NotebookServerImplFactory;
import io.hops.hopsworks.api.zeppelin.util.SecurityUtils;
import io.hops.hopsworks.common.jobs.jobhistory.JobType;
import io.hops.hopsworks.common.util.ConfigFileGenerator;
import io.hops.hopsworks.common.util.HopsUtils;
import io.hops.hopsworks.common.util.Settings;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.security.PrivilegedExceptionAction;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.helium.Helium;
import org.apache.zeppelin.helium.HeliumApplicationFactory;
import org.apache.zeppelin.helium.HeliumBundleFactory;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.InterpreterOutput;
import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.notebook.Notebook;
import org.apache.zeppelin.notebook.NotebookAuthorization;
import org.apache.zeppelin.notebook.repo.NotebookRepoSync;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.apache.zeppelin.search.LuceneSearch;
import org.apache.zeppelin.search.SearchService;
import org.apache.zeppelin.user.Credentials;
import org.quartz.SchedulerException;
import org.sonatype.aether.RepositoryException;

public class ZeppelinConfig {

    private static final Logger LOGGER = Logger.getLogger(ZeppelinConfig.class.getName());
    private static final String ZEPPELIN_SITE_XML = "/zeppelin-site.xml";
    private static final String ZEPPELIN_ENV_SH = "/zeppelin-env.sh";
    public static final String INTERPRETER_JSON = "/interpreter.json";
    private static final int DELETE_RETRY = 10;

    /**
     * A configuration that is common for all projects.
     */
    public static ZeppelinConfiguration COMMON_CONF;
    private ZeppelinConfiguration conf;
    private SchedulerFactory schedulerFactory;

    private Notebook notebook;
    private NotebookServerImpl notebookServer;
    private InterpreterFactory replFactory;
    private NotebookRepoSync notebookRepo;
    private DependencyResolver depResolver;
    private NotebookAuthorization notebookAuthorization;
    private InterpreterSettingManager interpreterSettingManager;
    private Helium helium;
    private HeliumApplicationFactory heliumApplicationFactory;
    private HeliumBundleFactory heliumBundleFactory;
    private Credentials credentials;
    private SearchService noteSearchService;
    private final Settings settings;
    private final String projectName;
    private final Integer projectId;
    private final String projectDirPath;
    private final String confDirPath;
    private final String notebookDirPath;
    private final String runDirPath;
    private final String binDirPath;
    private final String logDirPath;
    private final String interpreterDirPath;
    private final String libDirPath;
    private final String repoDirPath;
    private final String owner;

    public ZeppelinConfig(String projectName, Integer projectId, String owner, Settings settings,
            String interpreterConf, NotebookServerImpl nbs)
            throws IOException, RepositoryException, TaskRunnerException {
        this.projectName = projectName;
        this.projectId = projectId;
        this.owner = owner;
        this.settings = settings;
        this.projectDirPath = settings.getZeppelinDir() + File.separator + Settings.DIR_ROOT + File.separator
                + this.projectName;
        this.runDirPath = this.projectDirPath + File.separator + "run";
        this.binDirPath = this.projectDirPath + File.separator + "bin";
        this.logDirPath = this.projectDirPath + File.separator + "logs";
        this.libDirPath = this.projectDirPath + File.separator + "lib";
        this.confDirPath = this.projectDirPath + File.separator + "conf";
        this.interpreterDirPath = this.projectDirPath + File.separator + "interpreter";
        this.notebookDirPath = this.projectDirPath + File.separator + COMMON_CONF.getNotebookDir();
        this.repoDirPath = this.projectDirPath + File.separator + COMMON_CONF.getInterpreterLocalRepoPath();

        boolean newDir = false;
        boolean newFile = false;
        boolean newBinDir = false;
        try {
            newDir = createZeppelinDirs();//creates the necessary folders for the project in /srv/hops/zeppelin
            newBinDir = copyBinDir();
            createSymLinks();//interpreter and lib
            createVisCacheSymlink();//create a symlink to node and npm tar cache.
            newFile = createZeppelinConfFiles(interpreterConf);//create project specific configurations for zeppelin
            this.conf = loadConfig();//load the newly created zeppelin-site.xml
            this.depResolver = new DependencyResolver(
                    conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_LOCALREPO));
            InterpreterOutput.limit = conf.getInt(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT);
            this.heliumApplicationFactory = new HeliumApplicationFactory();
            this.heliumBundleFactory = new HeliumBundleFactory(conf, null,
                    new File(conf.getRelativeDir(ZeppelinConfiguration.ConfVars.ZEPPELIN_DEP_LOCALREPO)),
                    new File(conf.getRelativeDir("lib/node_modules/zeppelin-tabledata")),
                    new File(conf.getRelativeDir("lib/node_modules/zeppelin-vis")),
                    new File(conf.getRelativeDir("lib/node_modules/zeppelin-spell")));

            this.helium = new Helium(conf.getHeliumConfPath(), conf.getHeliumRegistry(),
                    new File(conf.getRelativeDir(ZeppelinConfiguration.ConfVars.ZEPPELIN_DEP_LOCALREPO),
                            "helium-registry-cache"),
                    heliumBundleFactory, heliumApplicationFactory);
            // create bundle
            try {
                this.heliumBundleFactory.buildAllPackages(helium.getBundlePackagesToBundle());
            } catch (Exception e) {
                LOGGER.log(Level.INFO, e.getMessage(), e);
            }

            this.schedulerFactory = SchedulerFactory.singleton();
            this.interpreterSettingManager = new InterpreterSettingManager(conf, depResolver,
                    new InterpreterOption(true));
            this.notebookRepo = getNotebookRepo(owner);
            this.noteSearchService = new LuceneSearch();
            this.notebookAuthorization = NotebookAuthorization.init(conf);
            this.credentials = new Credentials(conf.credentialsPersist(), conf.getCredentialsPath());
            if (nbs != null) {
                setNotebookServer(nbs);
            }
        } catch (IOException | RepositoryException | TaskRunnerException e) {
            if (newDir) { // if the folder was newly created delete it
                removeProjectDirRecursive();
            } else if (newFile) { // if the conf files were newly created delete them
                removeProjectConfFiles();
            }
            LOGGER.log(Level.SEVERE, "Error in initializing ZeppelinConfig for project: {0}. {1}",
                    new Object[] { this.projectName, e });
            throw e;
        }
    }

    public ZeppelinConfig(ZeppelinConfig zConf, NotebookServerImpl nbs) {
        this.settings = zConf.getSettings();
        this.projectName = zConf.getProjectName();
        this.projectId = zConf.getProjectId();
        this.owner = zConf.getOwner();
        this.projectDirPath = zConf.getProjectDirPath();
        this.confDirPath = zConf.getConfDirPath();
        this.notebookDirPath = zConf.getNotebookDirPath();
        this.runDirPath = zConf.getRunDirPath();
        this.binDirPath = zConf.getBinDirPath();
        this.logDirPath = zConf.getLogDirPath();
        this.interpreterDirPath = zConf.getInterpreterDirPath();
        this.libDirPath = zConf.getLibDirPath();
        this.repoDirPath = zConf.getRepoDirPath();
        this.conf = zConf.getConf();
        this.depResolver = zConf.getDepResolver();
        this.schedulerFactory = zConf.getSchedulerFactory();
        this.interpreterSettingManager = zConf.getInterpreterSettingManager();
        this.heliumApplicationFactory = zConf.getHeliumApplicationFactory();
        this.helium = zConf.getHelium();
        this.notebookRepo = zConf.getNotebookRepo();
        this.noteSearchService = zConf.getNotebookIndex();
        this.notebookAuthorization = zConf.getNotebookAuthorization();
        this.credentials = zConf.getCredentials();
        setNotebookServer(nbs);
    }

    private NotebookRepoSync getNotebookRepo(String owner) {
        if (owner == null) {
            return null;
        }
        String notebookStorage = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_STORAGE);
        LOGGER.log(Level.INFO, "Using notebook Repo class {0}", notebookStorage);
        boolean notebookInHdfs = notebookStorage.contains("HDFSNotebookRepo");
        if (!notebookInHdfs) {
            return new NotebookRepoSync(conf);
        }
        NotebookRepoSync nbRepo = null;
        UserGroupInformation ugi;
        try {
            ugi = UserGroupInformation.createProxyUser(owner, UserGroupInformation.getLoginUser());
            nbRepo = ugi.doAs((PrivilegedExceptionAction<NotebookRepoSync>) () -> new NotebookRepoSync(conf));
        } catch (IOException ex) {
            LOGGER.log(Level.SEVERE, "Could not create proxy user.", ex);
        } catch (InterruptedException ex) {
            LOGGER.log(Level.SEVERE, "Could not create notebook repo.", ex);
        }
        if (nbRepo == null) {
            LOGGER.log(Level.SEVERE, "Could not create notebook repo.");
            throw new IllegalStateException("Could not create notebook repo.");
        }
        return nbRepo;
    }

    public ZeppelinConfiguration getConf() {
        return this.conf;
    }

    public void setNotebookServer(NotebookServerImpl nbs) {
        if (this.notebookServer != null) {
            return;
        }
        this.notebookServer = nbs;
        try {
            this.replFactory = new InterpreterFactory(this.conf, this.notebookServer, this.notebookServer,
                    this.heliumApplicationFactory, this.depResolver, SecurityUtils.isAuthenticated(),
                    this.interpreterSettingManager);
            this.notebook = new Notebook(this.conf, this.notebookRepo, this.schedulerFactory, this.replFactory,
                    interpreterSettingManager, this.notebookServer, this.noteSearchService,
                    this.notebookAuthorization, this.credentials);
            // to update notebook from application event from remote process.
            this.heliumApplicationFactory.setNotebook(notebook);
            // to update fire websocket event on application event.
            this.heliumApplicationFactory.setApplicationEventListener(this.notebookServer);
            this.notebook.addNotebookEventListener(heliumApplicationFactory);
            this.notebook.addNotebookEventListener(this.notebookServer.getNotebookInformationListener());
        } catch (InterpreterException | IOException | RepositoryException | SchedulerException ex) {
            LOGGER.log(Level.SEVERE, null, ex);
        }
    }

    public boolean isClosed() {
        return this.replFactory == null || this.notebook == null || this.notebookServer == null;
    }

    public InterpreterSettingManager getInterpreterSettingManager() {
        return interpreterSettingManager;
    }

    public Notebook getNotebook() {
        return notebook;
    }

    public NotebookRepoSync getNotebookRepo() {
        return notebookRepo;
    }

    private SchedulerFactory getSchedulerFactory() {
        return this.schedulerFactory;
    }

    public Helium getHelium() {
        return helium;
    }

    private HeliumApplicationFactory getHeliumApplicationFactory() {
        return heliumApplicationFactory;
    }

    public NotebookServerImpl getNotebookServer() {
        return this.notebookServer;
    }

    public InterpreterFactory getReplFactory() {
        return this.replFactory;
    }

    public String getProjectName() {
        return projectName;
    }

    public Integer getProjectId() {
        return projectId;
    }

    public String getOwner() {
        return owner;
    }

    private String getProjectDirPath() {
        return projectDirPath;
    }

    public SearchService getNotebookIndex() {
        return noteSearchService;
    }

    private DependencyResolver getDepResolver() {
        return depResolver;
    }

    public void setDepResolver(DependencyResolver depResolver) {
        this.depResolver = depResolver;
    }

    public Settings getSettings() {
        return settings;
    }

    private String getInterpreterDirPath() {
        return interpreterDirPath;
    }

    private String getLibDirPath() {
        return libDirPath;
    }

    public String getConfDirPath() {
        return confDirPath;
    }

    private String getNotebookDirPath() {
        return notebookDirPath;
    }

    private String getRunDirPath() {
        return runDirPath;
    }

    private String getBinDirPath() {
        return binDirPath;
    }

    private String getLogDirPath() {
        return logDirPath;
    }

    private String getRepoDirPath() {
        return repoDirPath;
    }

    private NotebookAuthorization getNotebookAuthorization() {
        return notebookAuthorization;
    }

    private Credentials getCredentials() {
        return credentials;
    }

    //returns true if the project dir was created
    private boolean createZeppelinDirs() {
        File projectDir = new File(projectDirPath);
        boolean newProjectDir = projectDir.mkdirs();
        new File(confDirPath).mkdirs();
        new File(notebookDirPath).mkdirs();
        new File(runDirPath).mkdirs();
        new File(binDirPath).mkdirs();
        new File(logDirPath).mkdirs();
        new File(repoDirPath).mkdirs();
        return newProjectDir;
    }

    //copies /srv/zeppelin/bin to /srv/zeppelin/this.project/bin
    private boolean copyBinDir() throws IOException {
        String source = settings.getZeppelinDir() + File.separator + "bin";
        File binDir = new File(binDirPath);
        File sourceDir = new File(source);
        if (binDir.list().length == sourceDir.list().length) {
            //should probably check if the files are the same
            return false;
        }
        Path destinationPath;
        for (File file : sourceDir.listFiles()) {
            destinationPath = Paths.get(binDirPath + File.separator + file.getName());
            Files.copy(file.toPath(), destinationPath, StandardCopyOption.REPLACE_EXISTING);
        }
        return binDir.list().length == sourceDir.list().length;
    }

    //creates symlink to interpreters and libs
    private void createSymLinks() throws IOException {
        File target = new File(settings.getZeppelinDir() + File.separator + "interpreter");
        File newLink = new File(interpreterDirPath);
        if (!newLink.exists()) {
            Files.createSymbolicLink(newLink.toPath(), target.toPath());
        }
        target = new File(settings.getZeppelinDir() + File.separator + "lib");
        newLink = new File(libDirPath);
        if (!newLink.exists()) {
            Files.createSymbolicLink(newLink.toPath(), target.toPath());
        }
    }

    private void createVisCacheSymlink() throws IOException {
        File target = new File(settings.getZeppelinDir() + File.separator + "local-repo/vis");
        if (!target.exists()) {
            LOGGER.log(Level.SEVERE,
                    "Node and npm not cached at {0}. Zeppelin will try to download this for every project.",
                    target.toURI());
            return;
        }
        File newLink = new File(repoDirPath + File.separator + "vis");
        if (!newLink.exists()) {
            Files.createSymbolicLink(newLink.toPath(), target.toPath());
        }
    }

    // returns true if one of the conf files were created anew
    private boolean createZeppelinConfFiles(String interpreterConf) throws IOException {
        File zeppelin_env_file = new File(confDirPath + ZEPPELIN_ENV_SH);
        File zeppelin_site_xml_file = new File(confDirPath + ZEPPELIN_SITE_XML);
        File interpreter_file = new File(confDirPath + INTERPRETER_JSON);
        String home = settings.getZeppelinDir() + File.separator + Settings.DIR_ROOT + File.separator
                + this.projectName;
        String notebookDir = File.separator + Settings.DIR_ROOT + File.separator + this.projectName;
        String resourceDir = File.separator + Settings.DIR_ROOT + File.separator + this.projectName + File.separator
                + Settings.BaseDataset.RESOURCES.getName();
        boolean createdSh = false;
        boolean createdLog4j = false;
        boolean createdXml = false;

        String log4jPath = settings.getSparkLog4JPath();
        String zeppelinPythonPath = settings.getAnacondaProjectDir(this.projectName) + File.separator + "bin"
                + File.separator + "python";
        if (!zeppelin_env_file.exists()) {

            String ldLibraryPath = "";
            if (System.getenv().containsKey("LD_LIBRARY_PATH")) {
                ldLibraryPath = System.getenv("LD_LIBRARY_PATH");
            }
            String javaHome = settings.getJavaHome();
            if (System.getenv().containsKey("JAVA_HOME")) {
                javaHome = System.getenv("JAVA_HOME");
            }

            StringBuilder zeppelin_env = ConfigFileGenerator.instantiateFromTemplate(
                    ConfigFileGenerator.ZEPPELIN_ENV_TEMPLATE, "spark_dir", settings.getSparkDir(), "hadoop_dir",
                    settings.getHadoopSymbolicLinkDir(), "anaconda_env_dir",
                    settings.getAnacondaDir() + "/envs/" + this.projectName, "hadoop_username",
                    this.projectName + Settings.PROJECT_GENERIC_USER_SUFFIX, "java_home", javaHome, "cuda_dir",
                    settings.getCudaDir(), "ld_library_path", ldLibraryPath, "hadoop_classpath",
                    HopsUtils.getHadoopClasspathGlob(settings.getHadoopSymbolicLinkDir() + "/bin/hadoop",
                            "classpath", "--glob"),
                    "spark_options", "--files " + log4jPath);
            createdSh = ConfigFileGenerator.createConfigFile(zeppelin_env_file, zeppelin_env.toString());
        }

        if (!zeppelin_site_xml_file.exists()) {
            StringBuilder zeppelin_site_xml = ConfigFileGenerator.instantiateFromTemplate(
                    ConfigFileGenerator.ZEPPELIN_CONFIG_TEMPLATE, "zeppelin_home", home, "livy_url",
                    settings.getLivyUrl(), "livy_master", settings.getLivyYarnMode(), "zeppelin_home_dir", home,
                    "zeppelin_notebook_dir", notebookDir, "zeppelin_interpreters",
                    settings.getZeppelinInterpreters());
            createdXml = ConfigFileGenerator.createConfigFile(zeppelin_site_xml_file, zeppelin_site_xml.toString());
        }

        //get interpreter string from db
        if (interpreter_file.exists()) {
            interpreter_file.delete();
        }

        //Set Hopsworks properties to be available in Zeppelin
        String logstashID = "-D" + Settings.LOGSTASH_JOB_INFO + "=" + this.projectName.toLowerCase() + ",zeppelin,"
                + "notebook,?";
        String restEndpointProp = " -D" + Settings.HOPSWORKS_REST_ENDPOINT_PROPERTY + "="
                + settings.getRestEndpoint();
        String elasticEndpointProp = " -D" + Settings.HOPSWORKS_ELASTIC_ENDPOINT_PROPERTY + "="
                + settings.getElasticRESTEndpoint();
        String projectIdProp = " -D" + Settings.HOPSWORKS_PROJECTID_PROPERTY + "=" + this.projectId;
        String projectNameProp = " -D" + Settings.HOPSWORKS_PROJECTNAME_PROPERTY + "=" + this.projectName;
        String userProp = " -D" + Settings.HOPSWORKS_PROJECTUSER_PROPERTY + "=" + this.projectName
                + Settings.PROJECT_GENERIC_USER_SUFFIX;
        String jobType = " -D" + Settings.HOPSWORKS_JOBTYPE_PROPERTY + "=" + JobType.SPARK;
        String kafkaBrokers = " -D" + Settings.KAFKA_BROKERADDR_PROPERTY + "=" + settings.getKafkaBrokersStr();

        String extraJavaOptions = " -Dlog4j.configuration=./log4j.properties " + logstashID + restEndpointProp
                + elasticEndpointProp + projectIdProp + projectNameProp + userProp + jobType + kafkaBrokers;
        String hdfsResourceDir = "hdfs://" + resourceDir + File.separator;
        // Comma-separated files to be added as local resources to Spark/Livy interpreter
        String driverExtraClassPath = settings.getHopsLeaderElectionJarPath() + File.pathSeparator
                + settings.getHopsUtilFilename();
        String executorExtraClassPath = settings.getHopsLeaderElectionJarPath() + File.pathSeparator
                + settings.getHopsUtilFilename();

        StringBuilder distFiles = new StringBuilder();
        // When Hops RPC TLS is enabled, Yarn will take care of application certificate
        if (!settings.getHopsRpcTls()) {
            distFiles
                    // KeyStore
                    .append("hdfs://").append(settings.getHdfsTmpCertDir()).append(File.separator)
                    .append(projectName).append(Settings.PROJECT_GENERIC_USER_SUFFIX).append(File.separator)
                    .append(projectName).append(Settings.PROJECT_GENERIC_USER_SUFFIX).append("__kstore.jks#")
                    .append(Settings.K_CERTIFICATE).append(",")
                    // TrustStore
                    .append("hdfs://").append(settings.getHdfsTmpCertDir()).append(File.separator)
                    .append(projectName).append(Settings.PROJECT_GENERIC_USER_SUFFIX).append(File.separator)
                    .append(projectName).append(Settings.PROJECT_GENERIC_USER_SUFFIX).append("__tstore.jks#")
                    .append(Settings.T_CERTIFICATE).append(",")
                    // Material password
                    .append("hdfs://").append(settings.getHdfsTmpCertDir()).append(File.separator)
                    .append(projectName).append(Settings.PROJECT_GENERIC_USER_SUFFIX).append(File.separator)
                    .append(projectName).append(Settings.PROJECT_GENERIC_USER_SUFFIX).append("__cert.key#")
                    .append(Settings.CRYPTO_MATERIAL_PASSWORD).append(",");
        }
        distFiles
                // Glassfish domain truststore
                .append(settings.getGlassfishTrustStoreHdfs()).append("#").append(Settings.DOMAIN_CA_TRUSTSTORE)
                .append(",")
                // Add HopsUtil
                .append(settings.getHopsUtilHdfsPath()).append("#").append(settings.getHopsUtilFilename());

        if (interpreterConf == null) {
            StringBuilder interpreter_json = ConfigFileGenerator.instantiateFromTemplate(
                    ConfigFileGenerator.INTERPRETER_TEMPLATE, "projectName", this.projectName, "zeppelin_home_dir",
                    home, "hdfs_user", this.projectName + Settings.PROJECT_GENERIC_USER_SUFFIX, "hadoop_home",
                    settings.getHadoopSymbolicLinkDir(), "livy_url", settings.getLivyUrl(), "dist_files",
                    log4jPath + "," + distFiles.toString(), "extra_spark_java_options", extraJavaOptions,
                    "driver_extraClassPath", driverExtraClassPath, "executor_extraClassPath",
                    executorExtraClassPath, "spark.sql.warehouse.dir", hdfsResourceDir + "spark-warehouse",
                    "spark.yarn.stagingDir", hdfsResourceDir, "livy.spark.sql.warehouse.dir",
                    hdfsResourceDir + "spark-warehouse", "livy.spark.yarn.stagingDir", hdfsResourceDir,
                    "hadoop_version", settings.getHadoopVersion(), "zeppelin.python_conda_path", zeppelinPythonPath,
                    "livy_session_timeout", settings.getLivyZeppelinSessionTimeout(), "spark.yarn.dist.files",
                    distFiles.toString(), "hive.server", settings.getHiveServerHostName(false), "hive.db",
                    this.projectName);
            interpreterConf = interpreter_json.toString();
        }
        createdXml = ConfigFileGenerator.createConfigFile(interpreter_file, interpreterConf);

        return createdSh || createdXml || createdLog4j;
    }

    // loads configeration from project specific zeppelin-site.xml
    private ZeppelinConfiguration loadConfig() {
        URL url = null;
        File zeppelinConfig = new File(confDirPath + ZEPPELIN_SITE_XML);
        try {
            url = zeppelinConfig.toURI().toURL();
            LOGGER.log(Level.INFO, "Load configuration from {0}", url);
            conf = new ZeppelinConfiguration(url);
        } catch (ConfigurationException e) {
            LOGGER.log(Level.INFO, "Failed to load configuration from " + url + " proceeding with a default", e);
            conf = new ZeppelinConfiguration();
        } catch (MalformedURLException ex) {
            LOGGER.log(Level.INFO,
                    "Malformed URL failed to load configuration from " + url + " proceeding with a default", ex);
            conf = new ZeppelinConfiguration();
        }
        return conf;
    }

    /**
     * Closes all resources and deletes project dir
     * /srv/zeppelin/Projects/this.projectName recursive.
     *
     * @return true if the dir is deleted
     */
    public boolean cleanAndRemoveConfDirs(NotebookServerImplFactory notebookServerImplFactory) {
        clean(notebookServerImplFactory);
        return removeProjectDirRecursive();
    }

    private boolean removeProjectDirRecursive() {
        File projectDir = new File(projectDirPath);
        if (!projectDir.exists()) {
            return true;
        }
        boolean ret = false;
        File interpreter = new File(interpreterDirPath);
        File lib = new File(libDirPath);
        File repo = new File(repoDirPath + File.separator + "vis");
        //symlinks must be deleted before we recursive delete the project dir.
        int retry = 0;
        while (interpreter.exists() || lib.exists() || repo.exists()) {
            if (interpreter.exists()) {
                interpreter.delete();
            }
            if (lib.exists()) {
                lib.delete();
            }
            if (repo.exists()) {
                repo.delete();
            }
            retry++;
            if (retry > DELETE_RETRY) {
                LOGGER.log(Level.SEVERE, "Could not delete zeppelin project folder.");
                return false;
            }
        }
        try {
            ret = ConfigFileGenerator.deleteRecursive(projectDir);
        } catch (FileNotFoundException ex) {
        }
        return ret;
    }

    private boolean removeProjectConfFiles() {
        File zeppelin_env_file = new File(confDirPath + ZEPPELIN_ENV_SH);
        File zeppelin_site_xml_file = new File(confDirPath + ZEPPELIN_SITE_XML);
        File interpreter_file = new File(confDirPath + INTERPRETER_JSON);
        boolean ret = false;
        if (zeppelin_env_file.exists()) {
            ret = zeppelin_env_file.delete();
        }
        if (zeppelin_site_xml_file.exists()) {
            ret = zeppelin_site_xml_file.delete();
        }
        if (interpreter_file.exists()) {
            ret = interpreter_file.delete();
        }
        return ret;
    }

    /**
     * closes notebook SearchService, Repo, InterpreterFactory, and
     * SchedulerFactory
     */
    public void clean(NotebookServerImplFactory notebookServerImplFactory) {
        LOGGER.log(Level.INFO, "Cleanup of zeppelin resources for project {0}", this.projectName);
        if (interpreterSettingManager != null) {
            interpreterSettingManager.close();
        }
        // will close repo and index
        if (this.notebook != null) {
            this.notebook.close();
        }
        if (this.notebookServer != null) {
            this.notebookServer.closeConnections(notebookServerImplFactory);
        }
        this.schedulerFactory = null;
        this.replFactory = null;
        this.notebook = null;
    }
}