org.apache.zeppelin.interpreter.InterpreterSettingManager.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.zeppelin.interpreter.InterpreterSettingManager.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.zeppelin.interpreter;

import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.internal.StringMap;
import com.google.gson.reflect.TypeToken;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.lang.reflect.Type;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryStream.Filter;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;

import java.nio.file.attribute.PosixFilePermission;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.EnumSet;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.interpreter.Interpreter.RegisteredInterpreter;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.Job.Status;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.sonatype.aether.RepositoryException;
import org.sonatype.aether.repository.Authentication;
import org.sonatype.aether.repository.Proxy;
import org.sonatype.aether.repository.RemoteRepository;

/**
 * TBD
 */
public class InterpreterSettingManager {

    private static final Logger logger = LoggerFactory.getLogger(InterpreterSettingManager.class);
    private static final String SHARED_SESSION = "shared_session";
    private static final Map<String, Object> DEFAULT_EDITOR = ImmutableMap.of("language", (Object) "text",
            "editOnDblClick", false);

    private final ZeppelinConfiguration zeppelinConfiguration;
    private final Path interpreterDirPath;
    private final Path interpreterBindingPath;

    /**
     * This is only references with default settings, name and properties
     * key: InterpreterSetting.name
     */
    private final Map<String, InterpreterSetting> interpreterSettingsRef;
    /**
     * This is used by creating and running Interpreters
     * key: InterpreterSetting.id <- This is becuase backward compatibility
     */
    private final Map<String, InterpreterSetting> interpreterSettings;
    private final Map<String, List<String>> interpreterBindings;

    private final DependencyResolver dependencyResolver;
    private final List<RemoteRepository> interpreterRepositories;

    private final InterpreterOption defaultOption;

    private final Map<String, URLClassLoader> cleanCl;

    @Deprecated
    private String[] interpreterClassList;
    private String[] interpreterGroupOrderList;
    private InterpreterGroupFactory interpreterGroupFactory;

    private final Gson gson;

    public InterpreterSettingManager(ZeppelinConfiguration zeppelinConfiguration,
            DependencyResolver dependencyResolver, InterpreterOption interpreterOption)
            throws IOException, RepositoryException {
        this.zeppelinConfiguration = zeppelinConfiguration;
        this.interpreterDirPath = Paths.get(zeppelinConfiguration.getInterpreterDir());
        logger.debug("InterpreterRootPath: {}", interpreterDirPath);
        this.interpreterBindingPath = Paths.get(zeppelinConfiguration.getInterpreterSettingPath());
        logger.debug("InterpreterBindingPath: {}", interpreterBindingPath);

        this.interpreterSettingsRef = Maps.newConcurrentMap();
        this.interpreterSettings = Maps.newConcurrentMap();
        this.interpreterBindings = Maps.newConcurrentMap();

        this.dependencyResolver = dependencyResolver;
        this.interpreterRepositories = dependencyResolver.getRepos();

        this.defaultOption = interpreterOption;

        this.cleanCl = Collections.synchronizedMap(new HashMap<String, URLClassLoader>());

        String replsConf = zeppelinConfiguration.getString(ConfVars.ZEPPELIN_INTERPRETERS);
        this.interpreterClassList = replsConf.split(",");
        String groupOrder = zeppelinConfiguration.getString(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER);
        this.interpreterGroupOrderList = groupOrder.split(",");

        GsonBuilder gsonBuilder = new GsonBuilder();
        gsonBuilder.setPrettyPrinting();
        this.gson = gsonBuilder.create();

        init();
    }

    private void loadFromFile() {
        if (!Files.exists(interpreterBindingPath)) {
            // nothing to read
            return;
        }
        InterpreterInfoSaving infoSaving;
        try (BufferedReader json = Files.newBufferedReader(interpreterBindingPath, StandardCharsets.UTF_8)) {
            infoSaving = gson.fromJson(json, InterpreterInfoSaving.class);

            for (String k : infoSaving.interpreterSettings.keySet()) {
                InterpreterSetting setting = infoSaving.interpreterSettings.get(k);
                List<InterpreterInfo> infos = setting.getInterpreterInfos();

                // Convert json StringMap to Properties
                StringMap<String> p = (StringMap<String>) setting.getProperties();
                Properties properties = new Properties();
                for (String key : p.keySet()) {
                    properties.put(key, p.get(key));
                }
                setting.setProperties(properties);

                // Always use separate interpreter process
                // While we decided to turn this feature on always (without providing
                // enable/disable option on GUI).
                // previously created setting should turn this feature on here.
                setting.getOption().setRemote(true);

                // Update transient information from InterpreterSettingRef
                InterpreterSetting interpreterSettingObject = interpreterSettingsRef.get(setting.getGroup());
                if (interpreterSettingObject == null) {
                    logger.warn("can't get InterpreterSetting "
                            + "Information From loaded Interpreter Setting Ref - {} ", setting.getGroup());
                    continue;
                }
                String depClassPath = interpreterSettingObject.getPath();
                setting.setPath(depClassPath);

                for (InterpreterInfo info : infos) {
                    if (info.getEditor() == null) {
                        Map<String, Object> editor = getEditorFromSettingByClassName(interpreterSettingObject,
                                info.getClassName());
                        info.setEditor(editor);
                    }
                }

                setting.setInterpreterGroupFactory(interpreterGroupFactory);

                loadInterpreterDependencies(setting);
                interpreterSettings.put(k, setting);
            }

            interpreterBindings.putAll(infoSaving.interpreterBindings);

            if (infoSaving.interpreterRepositories != null) {
                for (RemoteRepository repo : infoSaving.interpreterRepositories) {
                    if (!dependencyResolver.getRepos().contains(repo)) {
                        this.interpreterRepositories.add(repo);
                    }
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    public void saveToFile() throws IOException {
        String jsonString;

        synchronized (interpreterSettings) {
            InterpreterInfoSaving info = new InterpreterInfoSaving();
            info.interpreterBindings = interpreterBindings;
            info.interpreterSettings = interpreterSettings;
            info.interpreterRepositories = interpreterRepositories;

            jsonString = gson.toJson(info);
        }

        if (!Files.exists(interpreterBindingPath)) {
            Files.createFile(interpreterBindingPath);

            Set<PosixFilePermission> permissions = EnumSet.of(OWNER_READ, OWNER_WRITE);
            Files.setPosixFilePermissions(interpreterBindingPath, permissions);
        }

        FileOutputStream fos = new FileOutputStream(interpreterBindingPath.toFile(), false);
        OutputStreamWriter out = new OutputStreamWriter(fos);
        out.append(jsonString);
        out.close();
        fos.close();
    }

    //TODO(jl): Fix it to remove InterpreterGroupFactory
    public void setInterpreterGroupFactory(InterpreterGroupFactory interpreterGroupFactory) {
        for (InterpreterSetting setting : interpreterSettings.values()) {
            setting.setInterpreterGroupFactory(interpreterGroupFactory);
        }
        this.interpreterGroupFactory = interpreterGroupFactory;
    }

    private void init() throws InterpreterException, IOException, RepositoryException {
        String interpreterJson = zeppelinConfiguration.getInterpreterJson();
        ClassLoader cl = Thread.currentThread().getContextClassLoader();

        if (Files.exists(interpreterDirPath)) {
            for (Path interpreterDir : Files.newDirectoryStream(interpreterDirPath, new Filter<Path>() {
                @Override
                public boolean accept(Path entry) throws IOException {
                    return Files.exists(entry) && Files.isDirectory(entry);
                }
            })) {
                String interpreterDirString = interpreterDir.toString();

                /**
                 * Register interpreter by the following ordering
                 * 1. Register it from path {ZEPPELIN_HOME}/interpreter/{interpreter_name}/
                 *    interpreter-setting.json
                 * 2. Register it from interpreter-setting.json in classpath
                 *    {ZEPPELIN_HOME}/interpreter/{interpreter_name}
                 * 3. Register it by Interpreter.register
                 */
                if (!registerInterpreterFromPath(interpreterDirString, interpreterJson)) {
                    if (!registerInterpreterFromResource(cl, interpreterDirString, interpreterJson)) {
                        /*
                         * TODO(jongyoul)
                         * - Remove these codes below because of legacy code
                         * - Support ThreadInterpreter
                        */
                        URLClassLoader ccl = new URLClassLoader(recursiveBuildLibList(interpreterDir.toFile()), cl);
                        for (String className : interpreterClassList) {
                            try {
                                // Load classes
                                Class.forName(className, true, ccl);
                                Set<String> interpreterKeys = Interpreter.registeredInterpreters.keySet();
                                for (String interpreterKey : interpreterKeys) {
                                    if (className.equals(Interpreter.registeredInterpreters.get(interpreterKey)
                                            .getClassName())) {
                                        Interpreter.registeredInterpreters.get(interpreterKey)
                                                .setPath(interpreterDirString);
                                        logger.info("Interpreter " + interpreterKey + " found. class=" + className);
                                        cleanCl.put(interpreterDirString, ccl);
                                    }
                                }
                            } catch (Throwable t) {
                                // nothing to do
                            }
                        }
                    }
                }
            }
        }

        for (RegisteredInterpreter registeredInterpreter : Interpreter.registeredInterpreters.values()) {
            logger.debug("Registered: {} -> {}. Properties: {}", registeredInterpreter.getInterpreterKey(),
                    registeredInterpreter.getClassName(), registeredInterpreter.getProperties());
        }

        // RegisteredInterpreters -> interpreterSettingRef
        InterpreterInfo interpreterInfo;
        for (RegisteredInterpreter r : Interpreter.registeredInterpreters.values()) {
            interpreterInfo = new InterpreterInfo(r.getClassName(), r.getName(), r.isDefaultInterpreter(),
                    r.getEditor());
            add(r.getGroup(), interpreterInfo, r.getProperties(), defaultOption, r.getPath(), r.getRunner());
        }

        for (String settingId : interpreterSettingsRef.keySet()) {
            InterpreterSetting setting = interpreterSettingsRef.get(settingId);
            logger.info("InterpreterSettingRef name {}", setting.getName());
        }

        loadFromFile();

        // if no interpreter settings are loaded, create default set
        if (0 == interpreterSettings.size()) {
            Map<String, InterpreterSetting> temp = new HashMap<>();
            InterpreterSetting interpreterSetting;
            for (InterpreterSetting setting : interpreterSettingsRef.values()) {
                interpreterSetting = createFromInterpreterSettingRef(setting);
                temp.put(setting.getName(), interpreterSetting);
            }

            for (String group : interpreterGroupOrderList) {
                if (null != (interpreterSetting = temp.remove(group))) {
                    interpreterSettings.put(interpreterSetting.getId(), interpreterSetting);
                }
            }

            for (InterpreterSetting setting : temp.values()) {
                interpreterSettings.put(setting.getId(), setting);
            }

            saveToFile();
        }

        for (String settingId : interpreterSettings.keySet()) {
            InterpreterSetting setting = interpreterSettings.get(settingId);
            logger.info("InterpreterSetting group {} : id={}, name={}", setting.getGroup(), settingId,
                    setting.getName());
        }
    }

    private boolean registerInterpreterFromResource(ClassLoader cl, String interpreterDir, String interpreterJson)
            throws IOException, RepositoryException {
        URL[] urls = recursiveBuildLibList(new File(interpreterDir));
        ClassLoader tempClassLoader = new URLClassLoader(urls, cl);

        Enumeration<URL> interpreterSettings = tempClassLoader.getResources(interpreterJson);
        if (!interpreterSettings.hasMoreElements()) {
            return false;
        }
        for (URL url : Collections.list(interpreterSettings)) {
            try (InputStream inputStream = url.openStream()) {
                logger.debug("Reading {} from {}", interpreterJson, url);
                List<RegisteredInterpreter> registeredInterpreterList = getInterpreterListFromJson(inputStream);
                registerInterpreters(registeredInterpreterList, interpreterDir);
            }
        }
        return true;
    }

    private boolean registerInterpreterFromPath(String interpreterDir, String interpreterJson)
            throws IOException, RepositoryException {

        Path interpreterJsonPath = Paths.get(interpreterDir, interpreterJson);
        if (Files.exists(interpreterJsonPath)) {
            logger.debug("Reading {}", interpreterJsonPath);
            List<RegisteredInterpreter> registeredInterpreterList = getInterpreterListFromJson(interpreterJsonPath);
            registerInterpreters(registeredInterpreterList, interpreterDir);
            return true;
        }
        return false;
    }

    private List<RegisteredInterpreter> getInterpreterListFromJson(Path filename) throws FileNotFoundException {
        return getInterpreterListFromJson(new FileInputStream(filename.toFile()));
    }

    private List<RegisteredInterpreter> getInterpreterListFromJson(InputStream stream) {
        Type registeredInterpreterListType = new TypeToken<List<RegisteredInterpreter>>() {
        }.getType();
        return gson.fromJson(new InputStreamReader(stream), registeredInterpreterListType);
    }

    private void registerInterpreters(List<RegisteredInterpreter> registeredInterpreters, String absolutePath)
            throws IOException, RepositoryException {

        for (RegisteredInterpreter registeredInterpreter : registeredInterpreters) {
            InterpreterInfo interpreterInfo = new InterpreterInfo(registeredInterpreter.getClassName(),
                    registeredInterpreter.getName(), registeredInterpreter.isDefaultInterpreter(),
                    registeredInterpreter.getEditor());
            // use defaultOption if it is not specified in interpreter-setting.json
            InterpreterOption option = registeredInterpreter.getOption() == null ? defaultOption
                    : registeredInterpreter.getOption();
            add(registeredInterpreter.getGroup(), interpreterInfo, registeredInterpreter.getProperties(), option,
                    absolutePath, registeredInterpreter.getRunner());
        }

    }

    public InterpreterSetting getDefaultInterpreterSetting(List<InterpreterSetting> settings) {
        if (settings == null || settings.isEmpty()) {
            return null;
        }
        return settings.get(0);
    }

    public InterpreterSetting getDefaultInterpreterSetting(String noteId) {
        return getDefaultInterpreterSetting(getInterpreterSettings(noteId));
    }

    public List<InterpreterSetting> getInterpreterSettings(String noteId) {
        List<String> interpreterSettingIds = getNoteInterpreterSettingBinding(noteId);
        LinkedList<InterpreterSetting> settings = new LinkedList<>();

        Iterator<String> iter = interpreterSettingIds.iterator();
        while (iter.hasNext()) {
            String id = iter.next();
            InterpreterSetting setting = get(id);
            if (setting == null) {
                // interpreter setting is removed from factory. remove id from here, too
                iter.remove();
            } else {
                settings.add(setting);
            }
        }
        return settings;
    }

    private List<String> getNoteInterpreterSettingBinding(String noteId) {
        LinkedList<String> bindings = new LinkedList<>();
        synchronized (interpreterSettings) {
            List<String> settingIds = interpreterBindings.get(noteId);
            if (settingIds != null) {
                bindings.addAll(settingIds);
            }
        }
        return bindings;
    }

    private InterpreterSetting createFromInterpreterSettingRef(String name) {
        Preconditions.checkNotNull(name, "reference name should be not null");
        InterpreterSetting settingRef = interpreterSettingsRef.get(name);
        return createFromInterpreterSettingRef(settingRef);
    }

    private InterpreterSetting createFromInterpreterSettingRef(InterpreterSetting o) {
        // should return immutable objects
        List<InterpreterInfo> infos = (null == o.getInterpreterInfos()) ? new ArrayList<InterpreterInfo>()
                : new ArrayList<>(o.getInterpreterInfos());
        List<Dependency> deps = (null == o.getDependencies()) ? new ArrayList<Dependency>()
                : new ArrayList<>(o.getDependencies());
        Properties props = convertInterpreterProperties((Map<String, InterpreterProperty>) o.getProperties());
        InterpreterOption option = InterpreterOption.fromInterpreterOption(o.getOption());

        InterpreterSetting setting = new InterpreterSetting(o.getName(), o.getName(), infos, props, deps, option,
                o.getPath(), o.getInterpreterRunner());
        setting.setInterpreterGroupFactory(interpreterGroupFactory);
        return setting;
    }

    private Properties convertInterpreterProperties(Map<String, InterpreterProperty> p) {
        Properties properties = new Properties();
        for (String key : p.keySet()) {
            properties.put(key, p.get(key).getValue());
        }
        return properties;
    }

    public Map<String, Object> getEditorSetting(Interpreter interpreter, String user, String noteId,
            String replName) {
        Map<String, Object> editor = DEFAULT_EDITOR;
        String group = StringUtils.EMPTY;
        try {
            String defaultSettingName = getDefaultInterpreterSetting(noteId).getName();
            List<InterpreterSetting> intpSettings = getInterpreterSettings(noteId);
            for (InterpreterSetting intpSetting : intpSettings) {
                String[] replNameSplit = replName.split("\\.");
                if (replNameSplit.length == 2) {
                    group = replNameSplit[0];
                }
                // when replName is 'name' of interpreter
                if (defaultSettingName.equals(intpSetting.getName())) {
                    editor = getEditorFromSettingByClassName(intpSetting, interpreter.getClassName());
                }
                // when replName is 'alias name' of interpreter or 'group' of interpreter
                if (replName.equals(intpSetting.getName()) || group.equals(intpSetting.getName())) {
                    editor = getEditorFromSettingByClassName(intpSetting, interpreter.getClassName());
                    break;
                }
            }
        } catch (NullPointerException e) {
            logger.warn("Couldn't get interpreter editor setting");
        }
        return editor;
    }

    public Map<String, Object> getEditorFromSettingByClassName(InterpreterSetting intpSetting, String className) {
        List<InterpreterInfo> intpInfos = intpSetting.getInterpreterInfos();
        for (InterpreterInfo intpInfo : intpInfos) {

            if (className.equals(intpInfo.getClassName())) {
                if (intpInfo.getEditor() == null) {
                    break;
                }
                return intpInfo.getEditor();
            }
        }
        return DEFAULT_EDITOR;
    }

    private void loadInterpreterDependencies(final InterpreterSetting setting) {
        setting.setStatus(InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES);
        setting.setErrorReason(null);
        interpreterSettings.put(setting.getId(), setting);
        synchronized (interpreterSettings) {
            final Thread t = new Thread() {
                public void run() {
                    try {
                        // dependencies to prevent library conflict
                        File localRepoDir = new File(
                                zeppelinConfiguration.getInterpreterLocalRepoPath() + "/" + setting.getId());
                        if (localRepoDir.exists()) {
                            try {
                                FileUtils.cleanDirectory(localRepoDir);
                            } catch (FileNotFoundException e) {
                                logger.info("A file that does not exist cannot be deleted, nothing to worry", e);
                            }
                        }

                        // load dependencies
                        List<Dependency> deps = setting.getDependencies();
                        if (deps != null) {
                            for (Dependency d : deps) {
                                File destDir = new File(
                                        zeppelinConfiguration.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO));

                                if (d.getExclusions() != null) {
                                    dependencyResolver.load(d.getGroupArtifactVersion(), d.getExclusions(),
                                            new File(destDir, setting.getId()));
                                } else {
                                    dependencyResolver.load(d.getGroupArtifactVersion(),
                                            new File(destDir, setting.getId()));
                                }
                            }
                        }

                        setting.setStatus(InterpreterSetting.Status.READY);
                        setting.setErrorReason(null);
                    } catch (Exception e) {
                        logger.error(String.format(
                                "Error while downloading repos for interpreter group : %s,"
                                        + " go to interpreter setting page click on edit and save it again to make "
                                        + "this interpreter work properly. : %s",
                                setting.getGroup(), e.getLocalizedMessage()), e);
                        setting.setErrorReason(e.getLocalizedMessage());
                        setting.setStatus(InterpreterSetting.Status.ERROR);
                    } finally {
                        interpreterSettings.put(setting.getId(), setting);
                    }
                }
            };
            t.start();
        }
    }

    /**
     * Overwrite dependency jar under local-repo/{interpreterId}
     * if jar file in original path is changed
     */
    private void copyDependenciesFromLocalPath(final InterpreterSetting setting) {
        setting.setStatus(InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES);
        interpreterSettings.put(setting.getId(), setting);
        synchronized (interpreterSettings) {
            final Thread t = new Thread() {
                public void run() {
                    try {
                        List<Dependency> deps = setting.getDependencies();
                        if (deps != null) {
                            for (Dependency d : deps) {
                                File destDir = new File(
                                        zeppelinConfiguration.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO));

                                int numSplits = d.getGroupArtifactVersion().split(":").length;
                                if (!(numSplits >= 3 && numSplits <= 6)) {
                                    dependencyResolver.copyLocalDependency(d.getGroupArtifactVersion(),
                                            new File(destDir, setting.getId()));
                                }
                            }
                        }
                        setting.setStatus(InterpreterSetting.Status.READY);
                    } catch (Exception e) {
                        logger.error(String.format("Error while copying deps for interpreter group : %s,"
                                + " go to interpreter setting page click on edit and save it again to make "
                                + "this interpreter work properly.", setting.getGroup()), e);
                        setting.setErrorReason(e.getLocalizedMessage());
                        setting.setStatus(InterpreterSetting.Status.ERROR);
                    } finally {
                        interpreterSettings.put(setting.getId(), setting);
                    }
                }
            };
            t.start();
        }
    }

    /**
     * Return ordered interpreter setting list.
     * The list does not contain more than one setting from the same interpreter class.
     * Order by InterpreterClass (order defined by ZEPPELIN_INTERPRETERS), Interpreter setting name
     */
    public List<String> getDefaultInterpreterSettingList() {
        // this list will contain default interpreter setting list
        List<String> defaultSettings = new LinkedList<>();

        // to ignore the same interpreter group
        Map<String, Boolean> interpreterGroupCheck = new HashMap<>();

        List<InterpreterSetting> sortedSettings = get();

        for (InterpreterSetting setting : sortedSettings) {
            if (defaultSettings.contains(setting.getId())) {
                continue;
            }

            if (!interpreterGroupCheck.containsKey(setting.getName())) {
                defaultSettings.add(setting.getId());
                interpreterGroupCheck.put(setting.getName(), true);
            }
        }
        return defaultSettings;
    }

    List<RegisteredInterpreter> getRegisteredInterpreterList() {
        return new ArrayList<>(Interpreter.registeredInterpreters.values());
    }

    private boolean findDefaultInterpreter(List<InterpreterInfo> infos) {
        for (InterpreterInfo interpreterInfo : infos) {
            if (interpreterInfo.isDefaultInterpreter()) {
                return true;
            }
        }
        return false;
    }

    public InterpreterSetting createNewSetting(String name, String group, List<Dependency> dependencies,
            InterpreterOption option, Properties p) throws IOException {
        if (name.indexOf(".") >= 0) {
            throw new IOException("'.' is invalid for InterpreterSetting name.");
        }
        InterpreterSetting setting = createFromInterpreterSettingRef(group);
        setting.setName(name);
        setting.setGroup(group);
        setting.appendDependencies(dependencies);
        setting.setInterpreterOption(option);
        setting.setProperties(p);
        setting.setInterpreterGroupFactory(interpreterGroupFactory);
        interpreterSettings.put(setting.getId(), setting);
        loadInterpreterDependencies(setting);
        saveToFile();
        return setting;
    }

    private InterpreterSetting add(String group, InterpreterInfo interpreterInfo,
            Map<String, InterpreterProperty> interpreterProperties, InterpreterOption option, String path,
            InterpreterRunner runner) throws InterpreterException, IOException, RepositoryException {
        ArrayList<InterpreterInfo> infos = new ArrayList<>();
        infos.add(interpreterInfo);
        return add(group, infos, new ArrayList<Dependency>(), option, interpreterProperties, path, runner);
    }

    /**
     * @param group InterpreterSetting reference name
     */
    public InterpreterSetting add(String group, ArrayList<InterpreterInfo> interpreterInfos,
            List<Dependency> dependencies, InterpreterOption option,
            Map<String, InterpreterProperty> interpreterProperties, String path, InterpreterRunner runner) {
        Preconditions.checkNotNull(group, "name should not be null");
        Preconditions.checkNotNull(interpreterInfos, "interpreterInfos should not be null");
        Preconditions.checkNotNull(dependencies, "dependencies should not be null");
        Preconditions.checkNotNull(option, "option should not be null");
        Preconditions.checkNotNull(interpreterProperties, "properties should not be null");

        InterpreterSetting interpreterSetting;

        synchronized (interpreterSettingsRef) {
            if (interpreterSettingsRef.containsKey(group)) {
                interpreterSetting = interpreterSettingsRef.get(group);

                // Append InterpreterInfo
                List<InterpreterInfo> infos = interpreterSetting.getInterpreterInfos();
                boolean hasDefaultInterpreter = findDefaultInterpreter(infos);
                for (InterpreterInfo interpreterInfo : interpreterInfos) {
                    if (!infos.contains(interpreterInfo)) {
                        if (!hasDefaultInterpreter && interpreterInfo.isDefaultInterpreter()) {
                            hasDefaultInterpreter = true;
                            infos.add(0, interpreterInfo);
                        } else {
                            infos.add(interpreterInfo);
                        }
                    }
                }

                // Append dependencies
                List<Dependency> dependencyList = interpreterSetting.getDependencies();
                for (Dependency dependency : dependencies) {
                    if (!dependencyList.contains(dependency)) {
                        dependencyList.add(dependency);
                    }
                }

                // Append properties
                Map<String, InterpreterProperty> properties = (Map<String, InterpreterProperty>) interpreterSetting
                        .getProperties();
                for (String key : interpreterProperties.keySet()) {
                    if (!properties.containsKey(key)) {
                        properties.put(key, interpreterProperties.get(key));
                    }
                }

            } else {
                interpreterSetting = new InterpreterSetting(group, null, interpreterInfos, interpreterProperties,
                        dependencies, option, path, runner);
                interpreterSettingsRef.put(group, interpreterSetting);
            }
        }

        if (dependencies.size() > 0) {
            loadInterpreterDependencies(interpreterSetting);
        }

        interpreterSetting.setInterpreterGroupFactory(interpreterGroupFactory);
        return interpreterSetting;
    }

    /**
     * map interpreter ids into noteId
     *
     * @param noteId note id
     * @param ids InterpreterSetting id list
     */
    public void setInterpreters(String user, String noteId, List<String> ids) throws IOException {
        putNoteInterpreterSettingBinding(user, noteId, ids);
    }

    private void putNoteInterpreterSettingBinding(String user, String noteId, List<String> settingList)
            throws IOException {
        List<String> unBindedSettings = new LinkedList<>();

        synchronized (interpreterSettings) {
            List<String> oldSettings = interpreterBindings.get(noteId);
            if (oldSettings != null) {
                for (String oldSettingId : oldSettings) {
                    if (!settingList.contains(oldSettingId)) {
                        unBindedSettings.add(oldSettingId);
                    }
                }
            }
            interpreterBindings.put(noteId, settingList);
            saveToFile();

            for (String settingId : unBindedSettings) {
                InterpreterSetting setting = get(settingId);
                removeInterpretersForNote(setting, user, noteId);
            }
        }
    }

    public void removeInterpretersForNote(InterpreterSetting interpreterSetting, String user, String noteId) {
        InterpreterOption option = interpreterSetting.getOption();
        if (option.isProcess()) {
            interpreterSetting.closeAndRemoveInterpreterGroupByNoteId(noteId);
        } else if (option.isSession()) {
            InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup(user, noteId);
            String key = getInterpreterSessionKey(user, noteId, interpreterSetting);
            interpreterGroup.close(key);
            synchronized (interpreterGroup) {
                interpreterGroup.remove(key);
                interpreterGroup.notifyAll(); // notify createInterpreterForNote()
            }
            logger.info("Interpreter instance {} for note {} is removed", interpreterSetting.getName(), noteId);
        }
    }

    public String getInterpreterSessionKey(String user, String noteId, InterpreterSetting setting) {
        InterpreterOption option = setting.getOption();
        String key;
        if (option.isExistingProcess()) {
            key = Constants.EXISTING_PROCESS;
        } else if (option.perNoteScoped() && option.perUserScoped()) {
            key = user + ":" + noteId;
        } else if (option.perUserScoped()) {
            key = user;
        } else if (option.perNoteScoped()) {
            key = noteId;
        } else {
            key = SHARED_SESSION;
        }

        logger.debug("Interpreter session key: {}, for note: {}, user: {}, InterpreterSetting Name: " + "{}", key,
                noteId, user, setting.getName());
        return key;
    }

    public List<String> getInterpreters(String noteId) {
        return getNoteInterpreterSettingBinding(noteId);
    }

    public void closeNote(String user, String noteId) {
        // close interpreters in this note session
        List<InterpreterSetting> settings = getInterpreterSettings(noteId);
        if (settings == null || settings.size() == 0) {
            return;
        }

        logger.info("closeNote: {}", noteId);
        for (InterpreterSetting setting : settings) {
            removeInterpretersForNote(setting, user, noteId);
        }
    }

    public Map<String, InterpreterSetting> getAvailableInterpreterSettings() {
        return interpreterSettingsRef;
    }

    private URL[] recursiveBuildLibList(File path) throws MalformedURLException {
        URL[] urls = new URL[0];
        if (path == null || !path.exists()) {
            return urls;
        } else if (path.getName().startsWith(".")) {
            return urls;
        } else if (path.isDirectory()) {
            File[] files = path.listFiles();
            if (files != null) {
                for (File f : files) {
                    urls = (URL[]) ArrayUtils.addAll(urls, recursiveBuildLibList(f));
                }
            }
            return urls;
        } else {
            return new URL[] { path.toURI().toURL() };
        }
    }

    public List<RemoteRepository> getRepositories() {
        return this.interpreterRepositories;
    }

    public void addRepository(String id, String url, boolean snapshot, Authentication auth, Proxy proxy)
            throws IOException {
        dependencyResolver.addRepo(id, url, snapshot, auth, proxy);
        saveToFile();
    }

    public void removeRepository(String id) throws IOException {
        dependencyResolver.delRepo(id);
        saveToFile();
    }

    public void removeNoteInterpreterSettingBinding(String user, String noteId) {
        synchronized (interpreterSettings) {
            List<String> settingIds = (interpreterBindings.containsKey(noteId) ? interpreterBindings.remove(noteId)
                    : Collections.<String>emptyList());
            for (String settingId : settingIds) {
                this.removeInterpretersForNote(get(settingId), user, noteId);
            }
        }
    }

    /**
     * Change interpreter property and restart
     */
    public void setPropertyAndRestart(String id, InterpreterOption option, Properties properties,
            List<Dependency> dependencies) throws IOException {
        synchronized (interpreterSettings) {
            InterpreterSetting intpSetting = interpreterSettings.get(id);
            if (intpSetting != null) {
                try {
                    stopJobAllInterpreter(intpSetting);

                    intpSetting.closeAndRemoveAllInterpreterGroups();
                    intpSetting.setOption(option);
                    intpSetting.setProperties(properties);
                    intpSetting.setDependencies(dependencies);
                    loadInterpreterDependencies(intpSetting);

                    saveToFile();
                } catch (Exception e) {
                    throw e;
                } finally {
                    loadFromFile();
                }
            } else {
                throw new InterpreterException("Interpreter setting id " + id + " not found");
            }
        }
    }

    public void restart(String settingId, String noteId, String user) {
        InterpreterSetting intpSetting = interpreterSettings.get(settingId);
        Preconditions.checkNotNull(intpSetting);

        // restart interpreter setting in note page
        if (noteIdIsExist(noteId) && intpSetting.getOption().isProcess()) {
            intpSetting.closeAndRemoveInterpreterGroupByNoteId(noteId);
            return;
        } else {
            // restart interpreter setting in interpreter setting page
            restart(settingId, user);
        }

    }

    private boolean noteIdIsExist(String noteId) {
        return noteId == null ? false : true;
    }

    public void restart(String id, String user) {
        synchronized (interpreterSettings) {
            InterpreterSetting intpSetting = interpreterSettings.get(id);
            // Check if dependency in specified path is changed
            // If it did, overwrite old dependency jar with new one
            if (intpSetting != null) {
                //clean up metaInfos
                intpSetting.setInfos(null);
                copyDependenciesFromLocalPath(intpSetting);

                stopJobAllInterpreter(intpSetting);
                if (user.equals("anonymous")) {
                    intpSetting.closeAndRemoveAllInterpreterGroups();
                } else {
                    intpSetting.closeAndRemoveInterpreterGroupByUser(user);
                }

            } else {
                throw new InterpreterException("Interpreter setting id " + id + " not found");
            }
        }
    }

    public void restart(String id) {
        restart(id, "anonymous");
    }

    private void stopJobAllInterpreter(InterpreterSetting intpSetting) {
        if (intpSetting != null) {
            for (InterpreterGroup intpGroup : intpSetting.getAllInterpreterGroups()) {
                for (List<Interpreter> interpreters : intpGroup.values()) {
                    for (Interpreter intp : interpreters) {
                        for (Job job : intp.getScheduler().getJobsRunning()) {
                            job.abort();
                            job.setStatus(Status.ABORT);
                            logger.info("Job " + job.getJobName() + " aborted ");
                        }
                        for (Job job : intp.getScheduler().getJobsWaiting()) {
                            job.abort();
                            job.setStatus(Status.ABORT);
                            logger.info("Job " + job.getJobName() + " aborted ");
                        }
                    }
                }
            }
        }
    }

    public InterpreterSetting get(String name) {
        synchronized (interpreterSettings) {
            return interpreterSettings.get(name);
        }
    }

    public void remove(String id) throws IOException {
        synchronized (interpreterSettings) {
            if (interpreterSettings.containsKey(id)) {
                InterpreterSetting intp = interpreterSettings.get(id);
                intp.closeAndRemoveAllInterpreterGroups();

                interpreterSettings.remove(id);
                for (List<String> settings : interpreterBindings.values()) {
                    Iterator<String> it = settings.iterator();
                    while (it.hasNext()) {
                        String settingId = it.next();
                        if (settingId.equals(id)) {
                            it.remove();
                        }
                    }
                }
                saveToFile();
            }
        }

        File localRepoDir = new File(zeppelinConfiguration.getInterpreterLocalRepoPath() + "/" + id);
        FileUtils.deleteDirectory(localRepoDir);
    }

    /**
     * Get interpreter settings
     */
    public List<InterpreterSetting> get() {
        synchronized (interpreterSettings) {
            List<InterpreterSetting> orderedSettings = new LinkedList<>();

            Map<String, List<InterpreterSetting>> nameInterpreterSettingMap = new HashMap<>();
            for (InterpreterSetting interpreterSetting : interpreterSettings.values()) {
                String group = interpreterSetting.getGroup();
                if (!nameInterpreterSettingMap.containsKey(group)) {
                    nameInterpreterSettingMap.put(group, new ArrayList<InterpreterSetting>());
                }
                nameInterpreterSettingMap.get(group).add(interpreterSetting);
            }

            for (String groupName : interpreterGroupOrderList) {
                List<InterpreterSetting> interpreterSettingList = nameInterpreterSettingMap.remove(groupName);
                if (null != interpreterSettingList) {
                    for (InterpreterSetting interpreterSetting : interpreterSettingList) {
                        orderedSettings.add(interpreterSetting);
                    }
                }
            }

            List<InterpreterSetting> settings = new ArrayList<>();

            for (List<InterpreterSetting> interpreterSettingList : nameInterpreterSettingMap.values()) {
                for (InterpreterSetting interpreterSetting : interpreterSettingList) {
                    settings.add(interpreterSetting);
                }
            }

            Collections.sort(settings, new Comparator<InterpreterSetting>() {
                @Override
                public int compare(InterpreterSetting o1, InterpreterSetting o2) {
                    return o1.getName().compareTo(o2.getName());
                }
            });

            orderedSettings.addAll(settings);

            return orderedSettings;
        }
    }

    public void close() {
        List<Thread> closeThreads = new LinkedList<>();
        synchronized (interpreterSettings) {
            Collection<InterpreterSetting> intpSettings = interpreterSettings.values();
            for (final InterpreterSetting intpSetting : intpSettings) {
                Thread t = new Thread() {
                    public void run() {
                        intpSetting.closeAndRemoveAllInterpreterGroups();
                    }
                };
                t.start();
                closeThreads.add(t);
            }
        }

        for (Thread t : closeThreads) {
            try {
                t.join();
            } catch (InterruptedException e) {
                logger.error("Can't close interpreterGroup", e);
            }
        }
    }

    public void shutdown() {
        List<Thread> closeThreads = new LinkedList<>();
        synchronized (interpreterSettings) {
            Collection<InterpreterSetting> intpSettings = interpreterSettings.values();
            for (final InterpreterSetting intpSetting : intpSettings) {
                Thread t = new Thread() {
                    public void run() {
                        intpSetting.shutdownAndRemoveAllInterpreterGroups();
                    }
                };
                t.start();
                closeThreads.add(t);
            }
        }

        for (Thread t : closeThreads) {
            try {
                t.join();
            } catch (InterruptedException e) {
                logger.error("Can't close interpreterGroup", e);
            }
        }
    }
}